diff --git a/.flake8 b/.flake8 index ebaca2e124..44033a966d 100644 --- a/.flake8 +++ b/.flake8 @@ -3,3 +3,4 @@ max-line-length = 88 ... select = C,E,F,W,B,B950 extend-ignore = E203, E501 +per-file-ignores = __init__.py:F401 diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index f64bfadd8a..0000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,15 +0,0 @@ -### Environmental Information - -- Operating System: -- Onadata version: -- Python version: - -### Problem description - -### Expected behavior - -### Steps to reproduce the behavior - -### Additional Information - -_Logs, [related issues](github.com/onaio/onadata/issues), weird / out of place occurrences, local settings, possible approach to solving this..._ diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature-request.md similarity index 100% rename from .github/ISSUE_TEMPLATE/feature_request.md rename to .github/ISSUE_TEMPLATE/feature-request.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 8d4b447778..b6c3be67e5 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -4,9 +4,10 @@ ### Side effects of implementing this change -### Before submitting this PR for review, please make sure you have: -- [ ] Included tests -- [ ] Updated documentation +**Before submitting this PR for review, please make sure you have:** + + - [ ] Included tests + - [ ] Updated documentation Closes # diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 736a82b0e8..4ed5223ec4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,14 +37,14 @@ jobs: - name: Setup Java uses: actions/setup-java@v2 with: - distribution: 'adopt' - java-version: '8' + distribution: "adopt" + java-version: "8" - name: Setup python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: - python-version: 3.8 - architecture: 'x64' + python-version: 3.9 + architecture: "x64" - name: Get pip cache dir id: pip-cache @@ -61,7 +61,7 @@ jobs: - name: Install APT requirements run: | sudo apt-get update - sudo apt-get install -y --no-install-recommends libjpeg-dev zlib1g-dev software-properties-common ghostscript libxslt1-dev binutils libproj-dev gdal-bin memcached libmemcached-dev + sudo apt-get install -y --no-install-recommends libjpeg-dev zlib1g-dev software-properties-common ghostscript libxslt1-dev binutils libproj-dev gdal-bin memcached libmemcached-dev libxml2-dev libxslt-dev sudo rm -rf /var/lib/apt/lists/* - name: Install Pip requirements @@ -69,6 +69,7 @@ jobs: pip install -U pip pip install -r requirements/base.pip pip install flake8 + pip install tblib - name: Run tests run: | diff --git a/.gitignore b/.gitignore index d58bfe49db..4947875f74 100644 --- a/.gitignore +++ b/.gitignore @@ -82,3 +82,7 @@ tags .bash_history .inputrc + +.eggs +sonar-project.properties +.scannerwork diff --git a/.prospector.yaml b/.prospector.yaml new file mode 100644 index 0000000000..452c66da60 --- /dev/null +++ b/.prospector.yaml @@ -0,0 +1,6 @@ +strictness: medium +doc-warnings: false +test-warnings: false +autodetect: true +member-warnings: false +max-line-length: 88 diff --git a/.pylintrc b/.pylintrc index a40e5f2066..ab8c6a1b0a 100644 --- a/.pylintrc +++ b/.pylintrc @@ -50,7 +50,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call +disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call,too-few-public-methods # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/CONTRIBUTING.MD b/CONTRIBUTING.MD index b3746493fc..c7b38f4b24 100644 --- a/CONTRIBUTING.MD +++ b/CONTRIBUTING.MD @@ -10,37 +10,37 @@ The following is a set of guidelines for contributing to Ona Data. Following the In case you have encountered any issue within the project, please make sure: -- You are using the [latest release](http://github.com/onaio/onadata/releases). -- You have setup the project according to the [Installation Documentation](https://api.ona.io/static/docs/install.html). + - You are using the [latest release](http://github.com/onaio/onadata/releases). + - You have setup the project according to the [Installation Documentation](https://api.ona.io/static/docs/install.html). After confirming the above, make sure the issue has not been reported on our [issues page](https://github.com/onaio/onadata/issues). If it hasn't been reported, [open a ticket](https://github.com/onaio/onadata/issues/new) containing: -- Information about your system environment (Operating System, local settings, etc.). -- What you expected to happen, and what actually happened. -- Out of place / weird logs and any other interesting information -- All steps to reproduce the issue. + - Information about your system environment (Operating System, local settings, etc.). + - What you expected to happen, and what actually happened. + - Out of place / weird logs and any other interesting information + - All steps to reproduce the issue. ### 2. Suggest Enhancements or New Features ⚡ Feature and enhancement requests are always welcome! We ask you ensure the following details are provided while [opening a ticket](https://github.com/onaio/onadata/issues/new), in order to start a constructive discussion: -- Describe the feature/enhancement in detail. -- Explain why the feature/enhancement is needed. -- Describe how the feature/enhancement should work -- List any advantages & disadvantages of implementing the feature/enhancement + - Describe the feature/enhancement in detail. + - Explain why the feature/enhancement is needed. + - Describe how the feature/enhancement should work + - List any advantages & disadvantages of implementing the feature/enhancement ### 3. Code contributions / Pull requests 💻 Pull requests are wholeheartedly welcome!❤️ If you are unsure about how to make your first pull request, here are some helpful resources: -- [Creating a pull request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request) -- [How to create effective pull requests](https://dev.to/mpermar/how-to-create-effective-pull-requests-2m8e) + - [Creating a pull request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request) + - [How to create effective pull requests](https://dev.to/mpermar/how-to-create-effective-pull-requests-2m8e) In order to make it easier for us to facilitate the smooth merging of your pull request, please make sure the following standards are met within your pull request. -- Ensure your git commits are signed. _Read about signing commits [here](https://help.github.com/en/github/authenticating-to-github/signing-commits)_ -- Code & commits follow our [styleguides](#Styleguides). -- Implement / Update tests that need to be updated and ensure that they pass. _Running all the tests within this project may be tough and time consuming. But not to worry! On submission of the pull request, [Travis CI](https://travis-ci.org/) will run all the tests across our modules_. + - Ensure your git commits are signed. _Read about signing commits [here](https://help.github.com/en/github/authenticating-to-github/signing-commits)_ + - Code & commits follow our [styleguides](#Styleguides). + - Implement / Update tests that need to be updated and ensure that they pass. _Running all the tests within this project may be tough and time consuming. But not to worry! On submission of the pull request, [Travis CI](https://travis-ci.org/) will run all the tests across our modules_. With the above points in mind feel free to comment on one of our [beginner-friendly issues](https://github.com/onaio/onadata/issues?q=is%3Aissue+is%3Aopen+label%3A%22Good+First+Issue%22) expressing your intent to work on it. diff --git a/docker-compose.yml b/docker-compose.yml index 5621ef2347..2ed3c383e1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: db: - image: postgis/postgis:9.6-3.0 + image: postgis/postgis:13-3.0 environment: - POSTGRES_PASSWORD=onadata - POSTGRES_USER=onadata @@ -29,7 +29,7 @@ services: - db - queue environment: - - SELECTED_PYTHON=python3.6 + - SELECTED_PYTHON=python3.9 - INITDB=false notifications: image: emqx/emqx:4.3.2 diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 12b2ac667c..446e5e9400 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/bash -if [ ${INITDB} ]; then +if [ "${INITDB}" ]; then RUN_DB_INIT_SCRIPT=$INITDB else RUN_DB_INIT_SCRIPT=true @@ -13,8 +13,8 @@ if $RUN_DB_INIT_SCRIPT; then psql -h db -U postgres onadata -c "CREATE EXTENSION postgis; CREATE EXTENSION postgis_topology;" fi -virtualenv -p `which $SELECTED_PYTHON` /srv/onadata/.virtualenv/${SELECTED_PYTHON} -. /srv/onadata/.virtualenv/${SELECTED_PYTHON}/bin/activate +virtualenv -p "$(which "${SELECTED_PYTHON}")" "/srv/onadata/.virtualenv/${SELECTED_PYTHON}" +. /srv/onadata/.virtualenv/"${SELECTED_PYTHON}"/bin/activate cd /srv/onadata pip install --upgrade pip diff --git a/docker/onadata-uwsgi/Dockerfile b/docker/onadata-uwsgi/Dockerfile index 5a4ea72037..f989abec35 100644 --- a/docker/onadata-uwsgi/Dockerfile +++ b/docker/onadata-uwsgi/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8 as intermediate +FROM python:3.9 as intermediate ENV DEBIAN_FRONTEND noninteractive ENV PYTHONUNBUFFERED 1 @@ -14,7 +14,7 @@ RUN mkdir -m 0600 ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts RUN --mount=type=ssh if [ -n "$optional_packages" ]; then pip install ${optional_packages} ; fi FROM ubuntu:20.04 -COPY --from=intermediate /usr/local/lib/python3.8/site-packages/ /usr/local/lib/python3.8/dist-packages/ +COPY --from=intermediate /usr/local/lib/python3.9/site-packages/ /usr/local/lib/python3.9/dist-packages/ ARG release_version=v2.4.1 @@ -35,8 +35,8 @@ RUN apt-get update -q &&\ libmemcached-dev \ build-essential \ supervisor \ - python3.8 \ - python3.8-dev \ + python3.9 \ + python3-dev \ python3-pip \ python3-setuptools \ git \ @@ -47,6 +47,7 @@ RUN apt-get update -q &&\ libjpeg-dev \ libxml2-dev \ libxslt1-dev \ + libpython3.9-dev \ zlib1g-dev \ ghostscript \ python3-celery \ @@ -79,11 +80,11 @@ COPY uwsgi.ini /uwsgi.ini # Install service requirements WORKDIR /srv/onadata # hadolint ignore=DL3013 -RUN python3 -m pip install --no-cache-dir -U pip && \ - python3 -m pip install --no-cache-dir -r requirements/base.pip && \ - python3 -m pip install --no-cache-dir -r requirements/s3.pip && \ - python3 -m pip install --no-cache-dir -r requirements/ses.pip && \ - python3 -m pip install --no-cache-dir uwsgitop django-prometheus==v2.2.0 +RUN python3.9 -m pip install --no-cache-dir -U pip && \ + python3.9 -m pip install --no-cache-dir -r requirements/base.pip && \ + python3.9 -m pip install --no-cache-dir -r requirements/s3.pip && \ + python3.9 -m pip install --no-cache-dir -r requirements/ses.pip && \ + python3.9 -m pip install --no-cache-dir pyyaml uwsgitop django-prometheus==v2.2.0 # Compile API Docs RUN make -C docs html diff --git a/docker/onadata-uwsgi/docker-compose.yml b/docker/onadata-uwsgi/docker-compose.yml index d881639e50..27ce6eb30b 100644 --- a/docker/onadata-uwsgi/docker-compose.yml +++ b/docker/onadata-uwsgi/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: db: - image: postgis/postgis:9.6-3.0 + image: postgis/postgis:13-3.0 environment: - POSTGRES_PASSWORD=onadata - POSTGRES_USER=onadata diff --git a/docker/postgis/Dockerfile b/docker/postgis/Dockerfile index b914a1372c..f2d491e497 100644 --- a/docker/postgis/Dockerfile +++ b/docker/postgis/Dockerfile @@ -1,12 +1,11 @@ FROM postgres:9.6 -MAINTAINER Ukang'a Dickson - - RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive \ - apt-get install -y postgresql-9.6-postgis-2.3 \ - postgresql-9.6-postgis-script postgis \ + apt-get install --no-install-recommends -y \ + postgresql-9.6-postgis-2.3=2.3.1+dfsg-2+deb9u2 \ + postgresql-9.6-postgis-2.3-scripts=2.3.1+dfsg-2+deb9u2 \ + postgis=2.3.1+dfsg-2+deb9u2 \ && rm -rf /var/lib/apt/lists/* ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/onadata/__init__.py b/onadata/__init__.py index e6f56e01e0..8616e0207a 100644 --- a/onadata/__init__.py +++ b/onadata/__init__.py @@ -6,7 +6,7 @@ """ from __future__ import absolute_import, unicode_literals -__version__ = "2.5.20" +__version__ = "3.0.0" # This will make sure the app is always imported when diff --git a/onadata/apps/api/admin.py b/onadata/apps/api/admin.py index d4a9ea6dbd..497104fd20 100644 --- a/onadata/apps/api/admin.py +++ b/onadata/apps/api/admin.py @@ -1,39 +1,38 @@ +# -*- coding: utf-8 -*- +"""API Django admin amendments.""" from django.contrib import admin from onadata.apps.api.models import Team, OrganizationProfile, TempToken -class TeamAdmin(admin.ModelAdmin): +# pylint: disable=too-few-public-methods +class FilterSuperuserMixin: + """Filter by request user and give full access to superuser.""" def get_queryset(self, request): - qs = super(TeamAdmin, self).get_queryset(request) + """Filter by request.user unless is_superuser.""" + queryset = super().get_queryset(request) if request.user.is_superuser: - return qs - return qs.filter(user=request.user) + return queryset + return queryset.filter(user=request.user) -admin.site.register(Team, TeamAdmin) +class TeamAdmin(FilterSuperuserMixin, admin.ModelAdmin): + """Filter by request.user unless is_superuser.""" -class OrganizationProfileAdmin(admin.ModelAdmin): +admin.site.register(Team, TeamAdmin) - def get_queryset(self, request): - qs = super(OrganizationProfileAdmin, self).get_queryset(request) - if request.user.is_superuser: - return qs - return qs.filter(user=request.user) +class OrganizationProfileAdmin(FilterSuperuserMixin, admin.ModelAdmin): + """Filter by request.user unless is_superuser.""" -admin.site.register(OrganizationProfile, OrganizationProfileAdmin) +admin.site.register(OrganizationProfile, OrganizationProfileAdmin) -class TempTokenProfileAdmin(admin.ModelAdmin): - def get_queryset(self, request): - qs = super(TempTokenProfileAdmin, self).get_queryset(request) - if request.user.is_superuser: - return qs - return qs.filter(user=request.user) +class TempTokenProfileAdmin(FilterSuperuserMixin, admin.ModelAdmin): + """Filter by request.user unless is_superuser.""" admin.site.register(TempToken, TempTokenProfileAdmin) diff --git a/onadata/apps/api/management/commands/fix_readonly_role_perms.py b/onadata/apps/api/management/commands/fix_readonly_role_perms.py index 9a76d0ce74..4e2eab79c4 100644 --- a/onadata/apps/api/management/commands/fix_readonly_role_perms.py +++ b/onadata/apps/api/management/commands/fix_readonly_role_perms.py @@ -1,32 +1,133 @@ +# -*- coding: utf-8 -*- +""" +fix_readonly_role_perms - Reassign permission to the model when permissions are changed +""" from guardian.shortcuts import get_perms from django.core.management.base import BaseCommand, CommandError -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.utils.translation import gettext as _ from django.conf import settings from onadata.apps.api.models import Team -from onadata.libs.permissions import ReadOnlyRole, DataEntryRole,\ - EditorRole, ManagerRole, OwnerRole, ReadOnlyRoleNoDownload,\ - DataEntryOnlyRole, DataEntryMinorRole, EditorMinorRole +from onadata.libs.permissions import ( + ReadOnlyRole, + DataEntryRole, + EditorRole, + ManagerRole, + OwnerRole, + ReadOnlyRoleNoDownload, + DataEntryOnlyRole, + DataEntryMinorRole, + EditorMinorRole, +) from onadata.libs.utils.model_tools import queryset_iterator +# pylint: disable=invalid-name +User = get_user_model() + + +def check_role(role_class, user, obj, new_perm=None): + """ + Test if the user has the role for the object provided + :param role_class: + :param user: + :param obj: + :param new_perm: + :return: + """ + new_perm = new_perm if new_perm is None else [] + # remove the new permission because the old model doesnt have it + perm_list = role_class.class_to_permissions[type(obj)] + old_perm_set = set(perm_list) + newly_added_perm = set(new_perm) + + if newly_added_perm.issubset(old_perm_set): + diff_set = old_perm_set.difference(newly_added_perm) + + if isinstance(user, Team): + return set(get_perms(user, obj)) == diff_set + + return user.has_perms(list(diff_set), obj) + return False + + +def reassign_perms(user, model, new_perm): + """ + Gets all the permissions the user has on objects and assigns the new + permission to them + :param user: + :param model: + :param new_perm: + :return: + """ + + # Get the unique permission model objects filtered by content type + # for the user + if isinstance(user, Team): + if model == "project": + objects = user.projectgroupobjectpermission_set.filter( + group_id=user.pk + ).distinct("content_object_id") + else: + objects = user.xformgroupobjectpermission_set.filter( + group_id=user.pk + ).distinct("content_object_id") + else: + if model == "project": + objects = user.projectuserobjectpermission_set.all() + else: + objects = user.xformuserobjectpermission_set.all() + + for perm_obj in objects: + obj = perm_obj.content_object + ROLES = [ + ReadOnlyRoleNoDownload, + ReadOnlyRole, + DataEntryOnlyRole, + DataEntryMinorRole, + DataEntryRole, + EditorMinorRole, + EditorRole, + ManagerRole, + OwnerRole, + ] + + # For each role reassign the perms + for role_class in reversed(ROLES): + not_readonly = role_class.user_has_role(user, obj) or role_class not in [ + ReadOnlyRoleNoDownload, + ReadOnlyRole, + ] + if not_readonly: + continue + + if check_role(role_class, user, obj, new_perm): + # If true + role_class.add(user, obj) + break + + class Command(BaseCommand): - args = '' - help = _(u"Reassign permission to the model when permissions are changed") + """ + fix_readonly_role_perms - Reassign permission to the model when + permissions are changed + """ + + args = "" + help = _("Reassign permission to the model when permissions are changed") def handle(self, *args, **options): - self.stdout.write("Re-assigining started", ending='\n') + self.stdout.write("Re-assigining started", ending="\n") if not args: - raise CommandError('Param not set. ') + raise CommandError("Param not set. ") if len(args) < 3: - raise CommandError('Param not set. ') + raise CommandError("Param not set. ") - app = args[0] model = args[1] username = args[2] new_perms = list(args[3:]) @@ -42,81 +143,9 @@ def handle(self, *args, **options): teams = Team.objects.filter(organization__username=username) # Get all the users for user in queryset_iterator(users): - self.reassign_perms(user, app, model, new_perms) + reassign_perms(user, model, new_perms) for team in queryset_iterator(teams): - self.reassign_perms(team, app, model, new_perms) - - self.stdout.write("Re-assigining finished", ending='\n') - - def reassign_perms(self, user, app, model, new_perm): - """ - Gets all the permissions the user has on objects and assigns the new - permission to them - :param user: - :param app: - :param model: - :param new_perm: - :return: - """ - - # Get the unique permission model objects filtered by content type - # for the user - if isinstance(user, Team): - if model == "project": - objects = user.projectgroupobjectpermission_set.filter( - group_id=user.pk).distinct('content_object_id') - else: - objects = user.xformgroupobjectpermission_set.filter( - group_id=user.pk).distinct('content_object_id') - else: - if model == 'project': - objects = user.projectuserobjectpermission_set.all() - else: - objects = user.xformuserobjectpermission_set.all() - - for perm_obj in objects: - obj = perm_obj.content_object - ROLES = [ReadOnlyRoleNoDownload, - ReadOnlyRole, - DataEntryOnlyRole, - DataEntryMinorRole, - DataEntryRole, - EditorMinorRole, - EditorRole, - ManagerRole, - OwnerRole] - - # For each role reassign the perms - for role_class in reversed(ROLES): - # want to only process for readonly perms - if role_class.user_has_role(user, obj) or role_class \ - not in [ReadOnlyRoleNoDownload, ReadOnlyRole]: - continue - - if self.check_role(role_class, user, obj, new_perm): - # If true - role_class.add(user, obj) - break - - def check_role(self, role_class, user, obj, new_perm=[]): - """ - Test if the user has the role for the object provided - :param role_class: - :param user: - :param obj: - :param new_perm: - :return: - """ - # remove the new permission because the old model doesnt have it - perm_list = role_class.class_to_permissions[type(obj)] - old_perm_set = set(perm_list) - newly_added_perm = set(new_perm) - - if newly_added_perm.issubset(old_perm_set): - diff_set = old_perm_set.difference(newly_added_perm) - - if isinstance(user, Team): - return set(get_perms(user, obj)) == diff_set - - return user.has_perms(list(diff_set), obj) + reassign_perms(team, model, new_perms) + + self.stdout.write("Re-assigining finished", ending="\n") diff --git a/onadata/apps/api/management/commands/increase_odk_token_lifetime.py b/onadata/apps/api/management/commands/increase_odk_token_lifetime.py index e98caeb687..d1e714719a 100644 --- a/onadata/apps/api/management/commands/increase_odk_token_lifetime.py +++ b/onadata/apps/api/management/commands/increase_odk_token_lifetime.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Command to increase the ODK token lifetime for a user. +""" from datetime import timedelta from django.core.management.base import BaseCommand, CommandError @@ -7,12 +11,14 @@ def increase_odk_token_lifetime(days: int, username: str): - qs = ODKToken.objects.filter( - user__username=username, status=ODKToken.ACTIVE) - if qs.count() < 1: + """ + Increase ODK Token lifetime for a particular user. + """ + queryset = ODKToken.objects.filter(user__username=username, status=ODKToken.ACTIVE) + if queryset.count() < 1: return False - token = qs.first() + token = queryset.first() updated_expiry_date = token.expires + timedelta(days=days) token.expires = updated_expiry_date.astimezone(token.expires.tzinfo) token.save() @@ -20,31 +26,34 @@ def increase_odk_token_lifetime(days: int, username: str): class Command(BaseCommand): + """ + increase_odk_token_lifetime command + + Increase ODK Token lifetime for a particular user. + """ + help = _("Increase ODK Token lifetime for a particular user.") def add_arguments(self, parser): parser.add_argument( - '--days', - '-d', + "--days", + "-d", default=30, - dest='days', - help='Number of days to increase the token lifetime by.') + dest="days", + help="Number of days to increase the token lifetime by.", + ) parser.add_argument( - '--username', - '-u', - dest='username', - help='The users username' + "--username", "-u", dest="username", help="The users username" ) def handle(self, *args, **options): - username = options.get('username') - days = options.get('days') + username = options.get("username") + days = options.get("days") if not username: - raise CommandError('No username provided.') + raise CommandError("No username provided.") created = increase_odk_token_lifetime(days, username) if not created: - raise CommandError(f'User {username} has no active ODK Token.') - self.stdout.write( - f'Increased the lifetime of ODK Token for user {username}') + raise CommandError(f"User {username} has no active ODK Token.") + self.stdout.write(f"Increased the lifetime of ODK Token for user {username}") diff --git a/onadata/apps/api/management/commands/reassign_permission.py b/onadata/apps/api/management/commands/reassign_permission.py index bd9dd10697..38ee347f10 100644 --- a/onadata/apps/api/management/commands/reassign_permission.py +++ b/onadata/apps/api/management/commands/reassign_permission.py @@ -1,30 +1,50 @@ +# -*- coding: utf-8 -*- +""" +reassign_permission - reassign permission to the model when permissions are changed. +""" from guardian.shortcuts import get_perms from django.core.management.base import BaseCommand, CommandError -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.utils.translation import gettext as _ from django.conf import settings from onadata.apps.api.models import Team -from onadata.libs.permissions import ReadOnlyRole, DataEntryRole,\ - EditorRole, ManagerRole, OwnerRole, ReadOnlyRoleNoDownload,\ - DataEntryOnlyRole, DataEntryMinorRole, EditorMinorRole +from onadata.libs.permissions import ( + ReadOnlyRole, + DataEntryRole, + EditorRole, + ManagerRole, + OwnerRole, + ReadOnlyRoleNoDownload, + DataEntryOnlyRole, + DataEntryMinorRole, + EditorMinorRole, +) from onadata.libs.utils.model_tools import queryset_iterator +# pylint: disable=invalid-name +User = get_user_model() + + class Command(BaseCommand): - args = '' - help = _(u"Reassign permission to the model when permissions are changed") + """Reassign permission to the model when permissions are changed""" + args = "" + help = _("Reassign permission to the model when permissions are changed") + + # pylint: disable=unused-argument def handle(self, *args, **options): - self.stdout.write("Re-assigining started", ending='\n') + """Reassign permission to the model when permissions are changed""" + self.stdout.write("Re-assigining started", ending="\n") if not args: - raise CommandError('Param not set. ') + raise CommandError("Param not set. ") if len(args) < 3: - raise CommandError('Param not set. ') + raise CommandError("Param not set. ") app = args[0] model = args[1] @@ -47,7 +67,7 @@ def handle(self, *args, **options): for team in queryset_iterator(teams): self.reassign_perms(team, app, model, new_perms) - self.stdout.write("Re-assigining finished", ending='\n') + self.stdout.write("Re-assigining finished", ending="\n") def reassign_perms(self, user, app, model, new_perm): """ @@ -65,37 +85,42 @@ def reassign_perms(self, user, app, model, new_perm): if isinstance(user, Team): if model == "project": objects = user.projectgroupobjectpermission_set.filter( - group_id=user.pk).distinct('content_object_id') + group_id=user.pk + ).distinct("content_object_id") else: objects = user.xformgroupobjectpermission_set.filter( - group_id=user.pk).distinct('content_object_id') + group_id=user.pk + ).distinct("content_object_id") else: - if model == 'project': + if model == "project": objects = user.projectuserobjectpermission_set.all() else: objects = user.xformuserobjectpermission_set.all() for perm_obj in objects: obj = perm_obj.content_object - ROLES = [ReadOnlyRoleNoDownload, - ReadOnlyRole, - DataEntryOnlyRole, - DataEntryMinorRole, - DataEntryRole, - EditorMinorRole, - EditorRole, - ManagerRole, - OwnerRole] + roles = [ + ReadOnlyRoleNoDownload, + ReadOnlyRole, + DataEntryOnlyRole, + DataEntryMinorRole, + DataEntryRole, + EditorMinorRole, + EditorRole, + ManagerRole, + OwnerRole, + ] # For each role reassign the perms - for role_class in reversed(ROLES): + for role_class in reversed(roles): if self.check_role(role_class, user, obj, new_perm): # If true role_class.add(user, obj) break - def check_role(self, role_class, user, obj, new_perm=[]): + # pylint: disable=no-self-use + def check_role(self, role_class, user, obj, new_perm=None): """ Test if the user has the role for the object provided :param role_class: @@ -104,6 +129,7 @@ def check_role(self, role_class, user, obj, new_perm=[]): :param new_perm: :return: """ + new_perm = [] if new_perm is None else new_perm # remove the new permission because the old model doesnt have it perm_list = role_class.class_to_permissions[type(obj)] old_perm_set = set(perm_list) @@ -116,3 +142,4 @@ def check_role(self, role_class, user, obj, new_perm=[]): return set(get_perms(user, obj)) == diff_set return user.has_perms(list(diff_set), obj) + return False diff --git a/onadata/apps/api/models/__init__.py b/onadata/apps/api/models/__init__.py index 6b89a4b54b..2366b4ba36 100644 --- a/onadata/apps/api/models/__init__.py +++ b/onadata/apps/api/models/__init__.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +""" +API models. +""" +# flake8: noqa + from onadata.apps.api.models.organization_profile import OrganizationProfile # noqa from onadata.apps.api.models.team import Team # noqa from onadata.apps.api.models.temp_token import TempToken # noqa diff --git a/onadata/apps/api/models/odk_token.py b/onadata/apps/api/models/odk_token.py index 8ab9e6b905..2d86886f7c 100644 --- a/onadata/apps/api/models/odk_token.py +++ b/onadata/apps/api/models/odk_token.py @@ -8,7 +8,7 @@ from django.conf import settings from django.db import models from django.db.models.signals import post_save -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from cryptography.fernet import Fernet from django_digest.models import (_persist_partial_digests, diff --git a/onadata/apps/api/models/organization_profile.py b/onadata/apps/api/models/organization_profile.py index fc58c95b81..d0dd1a3dad 100644 --- a/onadata/apps/api/models/organization_profile.py +++ b/onadata/apps/api/models/organization_profile.py @@ -6,7 +6,6 @@ from django.contrib.contenttypes.models import ContentType from django.db import models from django.db.models.signals import post_delete, post_save -from django.utils.encoding import python_2_unicode_compatible from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from guardian.shortcuts import assign_perm, get_perms_for_model @@ -23,7 +22,7 @@ def org_profile_post_delete_callback(sender, instance, **kwargs): """ # delete the org_user too instance.user.delete() - safe_delete('{}{}'.format(IS_ORG, instance.pk)) + safe_delete("{}{}".format(IS_ORG, instance.pk)) def create_owner_team_and_assign_permissions(org): @@ -32,14 +31,13 @@ def create_owner_team_and_assign_permissions(org): assigns the group and user permissions """ team = Team.objects.create( - name=Team.OWNER_TEAM_NAME, organization=org.user, - created_by=org.created_by) - content_type = ContentType.objects.get( - app_label='api', model='organizationprofile') + name=Team.OWNER_TEAM_NAME, organization=org.user, created_by=org.created_by + ) + content_type = ContentType.objects.get(app_label="api", model="organizationprofile") # pylint: disable=unpacking-non-sequence permission, _ = Permission.objects.get_or_create( - codename="is_org_owner", name="Organization Owner", - content_type=content_type) # pylint: disable= + codename="is_org_owner", name="Organization Owner", content_type=content_type + ) # pylint: disable= team.permissions.add(permission) org.creator.groups.add(team) @@ -53,23 +51,14 @@ def create_owner_team_and_assign_permissions(org): assign_perm(perm.codename, org.created_by, org) if org.userprofile_ptr: - for perm in get_perms_for_model( - org.userprofile_ptr.__class__): - assign_perm( - perm.codename, org.user, org.userprofile_ptr) + for perm in get_perms_for_model(org.userprofile_ptr.__class__): + assign_perm(perm.codename, org.user, org.userprofile_ptr) if org.creator: - assign_perm( - perm.codename, - org.creator, - org.userprofile_ptr) - - if org.created_by and\ - org.created_by != org.creator: - assign_perm( - perm.codename, - org.created_by, - org.userprofile_ptr) + assign_perm(perm.codename, org.creator, org.userprofile_ptr) + + if org.created_by and org.created_by != org.creator: + assign_perm(perm.codename, org.created_by, org.userprofile_ptr) return team @@ -83,25 +72,24 @@ def _post_save_create_owner_team(sender, instance, created, **kwargs): create_owner_team_and_assign_permissions(instance) -@python_2_unicode_compatible class OrganizationProfile(UserProfile): """Organization: Extends the user profile for organization specific info - * What does this do? - - it has a createor - - it has owner(s), through permissions/group - - has members, through permissions/group - - no login access, no password? no registration like a normal user? - - created by a user who becomes the organization owner - * What relationships? + * What does this do? + - it has a createor + - it has owner(s), through permissions/group + - has members, through permissions/group + - no login access, no password? no registration like a normal user? + - created by a user who becomes the organization owner + * What relationships? """ class Meta: - app_label = 'api' + app_label = "api" permissions = ( - ('can_add_project', "Can add a project to an organization"), - ('can_add_xform', "Can add/upload an xform to an organization") + ("can_add_project", "Can add a project to an organization"), + ("can_add_xform", "Can add/upload an xform to an organization"), ) is_organization = models.BooleanField(default=True) @@ -109,7 +97,7 @@ class Meta: creator = models.ForeignKey(User, on_delete=models.CASCADE) def __str__(self): - return u'%s[%s]' % (self.name, self.user.username) + return "%s[%s]" % (self.name, self.user.username) def save(self, *args, **kwargs): # pylint: disable=arguments-differ super(OrganizationProfile, self).save(*args, **kwargs) @@ -119,7 +107,7 @@ def remove_user_from_organization(self, user): :param user: The user to remove from this organization. """ - for group in user.groups.filter('%s#' % self.user.username): + for group in user.groups.filter("%s#" % self.user.username): user.groups.remove(group) def is_organization_owner(self, user): @@ -130,27 +118,32 @@ def is_organization_owner(self, user): :returns: Boolean whether user has organization level permissions. """ has_owner_group = user.groups.filter( - name='%s#%s' % (self.user.username, Team.OWNER_TEAM_NAME)) + name="%s#%s" % (self.user.username, Team.OWNER_TEAM_NAME) + ) return True if has_owner_group else False post_save.connect( - _post_save_create_owner_team, sender=OrganizationProfile, - dispatch_uid='create_owner_team_and_permissions') + _post_save_create_owner_team, + sender=OrganizationProfile, + dispatch_uid="create_owner_team_and_permissions", +) -post_delete.connect(org_profile_post_delete_callback, - sender=OrganizationProfile, - dispatch_uid='org_profile_post_delete_callback') +post_delete.connect( + org_profile_post_delete_callback, + sender=OrganizationProfile, + dispatch_uid="org_profile_post_delete_callback", +) # pylint: disable=model-no-explicit-unicode class OrgProfileUserObjectPermission(UserObjectPermissionBase): """Guardian model to create direct foreign keys.""" - content_object = models.ForeignKey( - OrganizationProfile, on_delete=models.CASCADE) + + content_object = models.ForeignKey(OrganizationProfile, on_delete=models.CASCADE) class OrgProfileGroupObjectPermission(GroupObjectPermissionBase): """Guardian model to create direct foreign keys.""" - content_object = models.ForeignKey( - OrganizationProfile, on_delete=models.CASCADE) + + content_object = models.ForeignKey(OrganizationProfile, on_delete=models.CASCADE) diff --git a/onadata/apps/api/models/team.py b/onadata/apps/api/models/team.py index 0297fceb8b..ad4a8e21ce 100644 --- a/onadata/apps/api/models/team.py +++ b/onadata/apps/api/models/team.py @@ -1,53 +1,65 @@ -from future.utils import python_2_unicode_compatible - +# -*- coding: utf-8 -*- +""" +Team model +""" from django.db import models from django.db.models.signals import post_save -from django.contrib.auth.models import User, Group +from django.contrib.auth import get_user_model +from django.contrib.auth.models import Group from guardian.shortcuts import assign_perm, get_perms_for_model from onadata.apps.logger.models.project import Project -@python_2_unicode_compatible +# pylint: disable=invalid-name +User = get_user_model() + + class Team(Group): """ - TODO: documentation - TODO: Whenever a member is removed from members team, - we should remove them from all teams and projects - within the organization. + Team model based on the Group. """ + class Meta: - app_label = 'api' + app_label = "api" OWNER_TEAM_NAME = "Owners" organization = models.ForeignKey(User, on_delete=models.CASCADE) projects = models.ManyToManyField(Project) created_by = models.ForeignKey( - User, related_name='team_creator', null=True, blank=True, - on_delete=models.SET_NULL) + User, + related_name="team_creator", + null=True, + blank=True, + on_delete=models.SET_NULL, + ) - date_created = models.DateTimeField(auto_now_add=True, null=True, - blank=True) + date_created = models.DateTimeField(auto_now_add=True, null=True, blank=True) date_modified = models.DateTimeField(auto_now=True, null=True, blank=True) def __str__(self): # return a clear group name without username to user for viewing - return self.name.split('#')[1] + return self.name.split("#")[1] @property def team_name(self): + """Return the team name.""" return self.__str__() def save(self, *args, **kwargs): # allow use of same name in different organizations/users # concat with # - if not self.name.startswith('#'.join([self.organization.username])): - self.name = u'%s#%s' % (self.organization.username, self.name) - super(Team, self).save(*args, **kwargs) + if not self.name.startswith("#".join([self.organization.username])): + self.name = f"{self.organization.username}#{self.name}" + super().save(*args, **kwargs) +# pylint: disable=unused-argument def set_object_permissions(sender, instance=None, created=False, **kwargs): + """ + Apply permissions to the creator of the team. + """ if created: for perm in get_perms_for_model(Team): assign_perm(perm.codename, instance.organization, instance) @@ -55,12 +67,15 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): if instance.created_by: assign_perm(perm.codename, instance.created_by, instance) - if hasattr(organization, 'creator') and \ - organization.creator != instance.created_by: + if ( + hasattr(organization, "creator") + and organization.creator != instance.created_by + ): assign_perm(perm.codename, organization.creator, instance) if organization.created_by != instance.created_by: assign_perm(perm.codename, organization.created_by, instance) -post_save.connect(set_object_permissions, sender=Team, - dispatch_uid='set_team_object_permissions') +post_save.connect( + set_object_permissions, sender=Team, dispatch_uid="set_team_object_permissions" +) diff --git a/onadata/apps/api/models/temp_token.py b/onadata/apps/api/models/temp_token.py index 0de4b6cc42..c9ded3c28e 100644 --- a/onadata/apps/api/models/temp_token.py +++ b/onadata/apps/api/models/temp_token.py @@ -5,33 +5,32 @@ import binascii import os -from django.conf import settings +from django.contrib.auth import get_user_model from django.db import models -from django.utils.encoding import python_2_unicode_compatible -AUTH_USER_MODEL = getattr(settings, 'AUTH_USER_MODEL', 'auth.User') - -@python_2_unicode_compatible class TempToken(models.Model): """ The temporary authorization token model. """ + key = models.CharField(max_length=40, primary_key=True) user = models.OneToOneField( - AUTH_USER_MODEL, related_name='_user', on_delete=models.CASCADE) + get_user_model(), related_name="_user", on_delete=models.CASCADE + ) created = models.DateTimeField(auto_now_add=True) class Meta: - app_label = 'api' + app_label = "api" def save(self, *args, **kwargs): if not self.key: self.key = self.generate_key() - return super(TempToken, self).save(*args, **kwargs) + return super().save(*args, **kwargs) - def generate_key(self): + def generate_key(self): # pylint: disable=no-self-use + """Generates a token key.""" return binascii.hexlify(os.urandom(20)).decode() def __str__(self): diff --git a/onadata/apps/api/tasks.py b/onadata/apps/api/tasks.py index b8e1f4ff2f..6f965e7943 100644 --- a/onadata/apps/api/tasks.py +++ b/onadata/apps/api/tasks.py @@ -7,7 +7,6 @@ from django.core.files.storage import default_storage from django.contrib.auth.models import User from django.utils.datastructures import MultiValueDict -from past.builtins import basestring from onadata.apps.api import tools from onadata.libs.utils.email import send_generic_email @@ -26,7 +25,7 @@ def recreate_tmp_file(name, path, mime_type): def publish_xlsform_async(self, user_id, post_data, owner_id, file_data): try: files = MultiValueDict() - files[u'xls_file'] = default_storage.open(file_data.get('path')) + files["xls_file"] = default_storage.open(file_data.get("path")) owner = User.objects.get(id=owner_id) if owner_id == user_id: @@ -34,7 +33,7 @@ def publish_xlsform_async(self, user_id, post_data, owner_id, file_data): else: user = User.objects.get(id=user_id) survey = tools.do_publish_xlsform(user, post_data, files, owner) - default_storage.delete(file_data.get('path')) + default_storage.delete(file_data.get("path")) if isinstance(survey, XForm): return {"pk": survey.pk} @@ -46,13 +45,13 @@ def publish_xlsform_async(self, user_id, post_data, owner_id, file_data): self.retry(exc=exc, countdown=1) else: error_message = ( - u'Service temporarily unavailable, please try to ' - 'publish the form again' + "Service temporarily unavailable, please try to " + "publish the form again" ) else: error_message = str(sys.exc_info()[1]) - return {u'error': error_message} + return {"error": error_message} @app.task() @@ -66,23 +65,23 @@ def delete_xform_async(xform_id, user_id): @app.task() def delete_user_async(): """Delete inactive user accounts""" - users = User.objects.filter(active=False, - username__contains="deleted-at", - email__contains="deleted-at") + users = User.objects.filter( + active=False, username__contains="deleted-at", email__contains="deleted-at" + ) for user in users: user.delete() def get_async_status(job_uuid): - """ Gets progress status or result """ + """Gets progress status or result""" if not job_uuid: - return {u'error': u'Empty job uuid'} + return {"error": "Empty job uuid"} job = AsyncResult(job_uuid) result = job.result or job.state - if isinstance(result, basestring): - return {'JOB_STATUS': result} + if isinstance(result, str): + return {"JOB_STATUS": result} return result diff --git a/onadata/apps/api/tests/fixtures/osm/combined.osm b/onadata/apps/api/tests/fixtures/osm/combined.osm index 3457809426..20ed3d01de 100644 --- a/onadata/apps/api/tests/fixtures/osm/combined.osm +++ b/onadata/apps/api/tests/fixtures/osm/combined.osm @@ -1,2 +1,2 @@ - + diff --git a/onadata/apps/api/tests/management/commands/test_create_user_profiles.py b/onadata/apps/api/tests/management/commands/test_create_user_profiles.py index 595d2226e0..e657b05999 100644 --- a/onadata/apps/api/tests/management/commands/test_create_user_profiles.py +++ b/onadata/apps/api/tests/management/commands/test_create_user_profiles.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- """Test create user profile management command.""" from django.contrib.auth.models import User -from onadata.apps.main.models.user_profile import UserProfile from django.core.management import call_command -from django.utils.six import StringIO +from six import StringIO + +from onadata.apps.main.models.user_profile import UserProfile from onadata.apps.main.tests.test_base import TestBase @@ -16,19 +17,13 @@ def test_create_user_profiles(self): successfully creates a user profile for users missing profiles. """ - user = User.objects.create( - username='dave', email='dave@example.com') + user = User.objects.create(username="dave", email="dave@example.com") with self.assertRaises(UserProfile.DoesNotExist): _ = user.profile out = StringIO() - call_command( - 'create_user_profiles', - stdout=out - ) + call_command("create_user_profiles", stdout=out) user.refresh_from_db() # Assert profile is retrievable; profile = user.profile self.assertEqual(profile.user, user) - self.assertEqual( - 'User Profiles successfully created.\n', - out.getvalue()) + self.assertEqual("User Profiles successfully created.\n", out.getvalue()) diff --git a/onadata/apps/api/tests/management/commands/test_delete_users.py b/onadata/apps/api/tests/management/commands/test_delete_users.py index 329e7eac89..ce975e13f5 100644 --- a/onadata/apps/api/tests/management/commands/test_delete_users.py +++ b/onadata/apps/api/tests/management/commands/test_delete_users.py @@ -3,59 +3,51 @@ """ import sys from unittest import mock -from django.utils.six import StringIO +from six import StringIO from django.contrib.auth.models import User from django.core.management import call_command from onadata.apps.main.tests.test_base import TestBase -from onadata.apps.api.management.commands.delete_users import \ - get_user_object_stats +from onadata.apps.api.management.commands.delete_users import get_user_object_stats class DeleteUserTest(TestBase): """ Test delete user management command. """ + def test_delete_users_with_input(self): """ Test that a user account is deleted automatically when the user_input field is provided as true """ - user = User.objects.create( - username="bruce", - email="bruce@gmail.com") + user = User.objects.create(username="bruce", email="bruce@gmail.com") username = user.username email = user.email out = StringIO() sys.stdout = out - new_user_details = [username+':'+email] + new_user_details = [username + ":" + email] call_command( - 'delete_users', - user_details=new_user_details, - user_input='True', - stdout=out) + "delete_users", user_details=new_user_details, user_input="True", stdout=out + ) - self.assertEqual( - "User bruce deleted successfully.", - out.getvalue()) + self.assertEqual("User bruce deleted successfully.", out.getvalue()) with self.assertRaises(User.DoesNotExist): User.objects.get(email="bruce@gmail.com") - @mock.patch( - "onadata.apps.api.management.commands.delete_users.input") + @mock.patch("onadata.apps.api.management.commands.delete_users.input") def test_delete_users_no_input(self, mock_input): # pylint: disable=R0201 """ Test that when user_input is not provided, the user account stats are provided for that user account before deletion """ - user = User.objects.create( - username="barbie", - email="barbie@gmail.com") + user = User.objects.create(username="barbie", email="barbie@gmail.com") username = user.username get_user_object_stats(username) mock_input.assert_called_with( "User account 'barbie' has 0 projects, " "0 forms and 0 submissions. " - "Do you wish to continue deleting this account?") + "Do you wish to continue deleting this account?" + ) diff --git a/onadata/apps/api/tests/management/commands/test_increase_odk_token_lifetime.py b/onadata/apps/api/tests/management/commands/test_increase_odk_token_lifetime.py index 3cd040d68a..c4936b72d0 100644 --- a/onadata/apps/api/tests/management/commands/test_increase_odk_token_lifetime.py +++ b/onadata/apps/api/tests/management/commands/test_increase_odk_token_lifetime.py @@ -1,30 +1,41 @@ +# -*- coding: utf-8 -*- +""" +Test increase_odk_token_lifetime command +""" from datetime import timedelta -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.management import call_command -from django.utils.six import StringIO -from onadata.apps.main.tests.test_base import TestBase +from six import StringIO + from onadata.apps.api.models.odk_token import ODKToken +from onadata.apps.main.tests.test_base import TestBase class IncreaseODKTokenLifetimeTest(TestBase): + """ + Test increase_odk_token_lifetime command + """ + + # pylint: disable=invalid-name def test_increase_odk_token_lifetime(self): - user = User.objects.create( - username='dave', email='dave@example.com') + """ + Test increase_odk_token_lifetime command + """ + user = get_user_model().objects.create( + username="dave", email="dave@example.com" + ) token = ODKToken.objects.create(user=user) expiry_date = token.expires out = StringIO() call_command( - 'increase_odk_token_lifetime', - days=2, - username=user.username, - stdout=out + "increase_odk_token_lifetime", days=2, username=user.username, stdout=out ) self.assertEqual( - 'Increased the lifetime of ODK Token for user dave\n', - out.getvalue()) + "Increased the lifetime of ODK Token for user dave\n", out.getvalue() + ) token.refresh_from_db() self.assertEqual(expiry_date + timedelta(days=2), token.expires) diff --git a/onadata/apps/api/tests/management/commands/test_retrieve_org_or_project_list.py b/onadata/apps/api/tests/management/commands/test_retrieve_org_or_project_list.py index 1fe1a383e7..703f838dfe 100644 --- a/onadata/apps/api/tests/management/commands/test_retrieve_org_or_project_list.py +++ b/onadata/apps/api/tests/management/commands/test_retrieve_org_or_project_list.py @@ -1,11 +1,10 @@ import json + from django.core.management import call_command -from django.utils.six import StringIO +from six import StringIO -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet -from onadata.libs.serializers.share_project_serializer import \ - ShareProjectSerializer +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet +from onadata.libs.serializers.share_project_serializer import ShareProjectSerializer class TestRetrieveOrgOrProjectListCommand(TestAbstractViewSet): @@ -13,52 +12,48 @@ def test_retrieve_org_or_project_list(self): self._org_create() self._project_create() - user = self._create_user_profile( - {'username': 'alice'}).user + user = self._create_user_profile({"username": "alice"}).user share_data = { - 'project': self.project.id, - 'username': user.username, - 'role': 'editor' + "project": self.project.id, + "username": user.username, + "role": "editor", } serializer = ShareProjectSerializer(data=share_data) self.assertTrue(serializer.is_valid()) serializer.save() out = StringIO() - call_command( - 'retrieve_org_or_project_list', - stdout=out - ) + call_command("retrieve_org_or_project_list", stdout=out) expected_project_data = { self.project.name: { user.username: { - 'first_name': user.first_name, - 'last_name': user.last_name, - 'is_org': False, - 'role': 'editor' + "first_name": user.first_name, + "last_name": user.last_name, + "is_org": False, + "role": "editor", }, self.user.username: { - 'first_name': self.user.first_name, - 'last_name': self.user.last_name, - 'is_org': False, - 'role': 'owner' - } + "first_name": self.user.first_name, + "last_name": self.user.last_name, + "is_org": False, + "role": "owner", + }, } } expected_org_data = { self.organization.name: { self.user.username: { - 'first_name': self.user.first_name, - 'last_name': self.user.last_name, - 'role': 'owner' + "first_name": self.user.first_name, + "last_name": self.user.last_name, + "role": "owner", }, self.organization.user.username: { - 'first_name': self.organization.user.first_name, - 'last_name': self.organization.user.last_name, - 'role': 'owner' - } + "first_name": self.organization.user.first_name, + "last_name": self.organization.user.last_name, + "role": "owner", + }, } } @@ -66,31 +61,20 @@ def test_retrieve_org_or_project_list(self): expected_data.update(expected_project_data) expected_data.update(expected_org_data) - self.assertEqual( - expected_data, - json.loads(out.getvalue()) - ) + self.assertEqual(expected_data, json.loads(out.getvalue())) out = StringIO() call_command( - 'retrieve_org_or_project_list', - project_ids=f'{self.project.id}', - stdout=out - ) - self.assertEqual( - expected_project_data, - json.loads(out.getvalue()) + "retrieve_org_or_project_list", project_ids=f"{self.project.id}", stdout=out ) + self.assertEqual(expected_project_data, json.loads(out.getvalue())) out = StringIO() call_command( - 'retrieve_org_or_project_list', - organization_ids=f'{self.organization.id}', - stdout=out - ) - self.assertEqual( - expected_org_data, - json.loads(out.getvalue()) + "retrieve_org_or_project_list", + organization_ids=f"{self.organization.id}", + stdout=out, ) + self.assertEqual(expected_org_data, json.loads(out.getvalue())) diff --git a/onadata/apps/api/tests/models/test_project.py b/onadata/apps/api/tests/models/test_project.py index 2016fa2110..283476e89d 100644 --- a/onadata/apps/api/tests/models/test_project.py +++ b/onadata/apps/api/tests/models/test_project.py @@ -45,9 +45,9 @@ def test_project_soft_delete_works_when_no_exception_is_raised(self): project=project ) project.soft_delete() - self.assertEquals( + self.assertEqual( 1, Project.objects.filter(deleted_at__isnull=False).count()) - self.assertEquals( + self.assertEqual( 1, XForm.objects.filter(deleted_at__isnull=False).count()) def test_project_detetion_reverts_when_an_exception_raised(self): @@ -100,16 +100,16 @@ def test_project_detetion_reverts_when_an_exception_raised(self): with self.assertRaises(XLSFormError): project.soft_delete() - self.assertEquals(1, Project.objects.filter( + self.assertEqual(1, Project.objects.filter( deleted_at__isnull=True).count()) self.assertIsNone(project.deleted_at) - self.assertEquals(1, XForm.objects.filter( + self.assertEqual(1, XForm.objects.filter( project=project, deleted_at__isnull=True).count()) # Try deleting the Xform; it should also roll back due to the exception with self.assertRaises(XLSFormError): XForm.objects.all()[0].soft_delete() - self.assertEquals(1, XForm.objects.filter( + self.assertEqual(1, XForm.objects.filter( deleted_at__isnull=True).count()) self.assertIsNone(XForm.objects.all()[0].deleted_at) diff --git a/onadata/apps/api/tests/models/test_temp_token.py b/onadata/apps/api/tests/models/test_temp_token.py index 95bff92a88..8a262bf817 100644 --- a/onadata/apps/api/tests/models/test_temp_token.py +++ b/onadata/apps/api/tests/models/test_temp_token.py @@ -14,7 +14,7 @@ def test_temp_token_creation(self): self.assertEqual(token.key, '456c27c7d59303aed1dffd3b5ffaad36f0676618') self.assertTrue(created) - self.assertEquals(initial_count + 1, TempToken.objects.count()) + self.assertEqual(initial_count + 1, TempToken.objects.count()) # update initial count initial_count = TempToken.objects.count() @@ -35,7 +35,7 @@ def test_temp_token_creation(self): TempToken.objects.get( user=self.user, key='456c27c7d59303aed1dffd3b5ffaad36f0676618').delete() - self.assertEquals(initial_count - 1, TempToken.objects.count()) + self.assertEqual(initial_count - 1, TempToken.objects.count()) token1, created1 = TempToken.objects.get_or_create( user=self.user, key='1b3b82a23063a8a4e64fb4434dc21ab181fbbe7c') diff --git a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py index 8d028fd7dd..b4a1e8b7a1 100644 --- a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py @@ -1,17 +1,22 @@ +# -*- coding: utf-8 -*- +""" +Test base class for API viewset tests. +""" import json import os import re from tempfile import NamedTemporaryFile -from builtins import open +import requests + from django.conf import settings -from django.contrib.auth import authenticate -from django.contrib.auth.models import Permission, User +from django.contrib.auth import authenticate, get_user_model +from django.contrib.auth.models import Permission from django.test import TestCase + from django_digest.test import Client as DigestClient from django_digest.test import DigestAuth from httmock import HTTMock -from onadata.libs.test_utils.pyxform_test_case import PyxformMarkdown from rest_framework.test import APIRequestFactory from onadata.apps.api.models import OrganizationProfile, Team @@ -32,11 +37,67 @@ from onadata.apps.main.models import MetaData, UserProfile from onadata.apps.viewer.models import DataDictionary from onadata.libs.serializers.project_serializer import ProjectSerializer +from onadata.libs.test_utils.pyxform_test_case import PyxformMarkdown from onadata.libs.utils.common_tools import merge_dicts from onadata.libs.utils.user_auth import get_user_default_project +# pylint: disable=invalid-name +User = get_user_model() + + +def _set_api_permissions(user): + add_userprofile = Permission.objects.get( + content_type__app_label="main", + content_type__model="userprofile", + codename="add_userprofile", + ) + user.user_permissions.add(add_userprofile) + + +def add_uuid_to_submission_xml(path, xform): + """ + Adds the formhub uuid to an XML XForm submission at the given path. + """ + with NamedTemporaryFile(delete=False, mode="w") as tmp_file: + split_xml = None + + with open(path, encoding="utf-8") as _file: + split_xml = re.split(r"()", _file.read()) + + split_xml[1:1] = [f"{xform.uuid}"] + tmp_file.write("".join(split_xml)) + path = tmp_file.name + + return path + + +# pylint: disable=invalid-name +def get_mocked_response_for_file(file_object, filename, status_code=200): + """Returns a requests.Response() object for mocked tests.""" + mock_response = requests.Response() + mock_response.status_code = status_code + mock_response.headers = { + "content-type": ( + "application/vnd.openxmlformats-" "officedocument.spreadsheetml.sheet" + ), + "Content-Disposition": ( + 'attachment; filename="transportation.' + f"xlsx\"; filename*=UTF-8''{filename}" + ), + } + # pylint: disable=protected-access + mock_response._content = file_object.read() + + return mock_response + + +# pylint: disable=too-many-instance-attributes class TestAbstractViewSet(PyxformMarkdown, TestCase): + """ + Base test class for API viewsets. + """ + surveys = [ "transport_2011-07-25_19-05-49", "transport_2011-07-25_19-05-36", @@ -67,6 +128,7 @@ def setUp(self): self.maxDiff = None def user_profile_data(self): + """Returns the user profile python object.""" return { "id": self.user.pk, "url": "http://testserver/api/v1/profiles/bob", @@ -88,17 +150,10 @@ def user_profile_data(self): "name": "Bob erama", } - def _set_api_permissions(self, user): - add_userprofile = Permission.objects.get( - content_type__app_label="main", - content_type__model="userprofile", - codename="add_userprofile", - ) - user.user_permissions.add(add_userprofile) - - def _create_user_profile(self, extra_post_data={}): + def _create_user_profile(self, extra_post_data=None): + extra_post_data = {} if extra_post_data is None else extra_post_data self.profile_data = merge_dicts(self.profile_data, extra_post_data) - user, created = User.objects.get_or_create( + user, _created = User.objects.get_or_create( username=self.profile_data["username"], first_name=self.profile_data["first_name"], last_name=self.profile_data["last_name"], @@ -106,7 +161,7 @@ def _create_user_profile(self, extra_post_data={}): ) user.set_password(self.profile_data["password1"]) user.save() - new_profile, created = UserProfile.objects.get_or_create( + new_profile, _created = UserProfile.objects.get_or_create( user=user, name=self.profile_data["first_name"], city=self.profile_data["city"], @@ -119,7 +174,8 @@ def _create_user_profile(self, extra_post_data={}): return new_profile - def _login_user_and_profile(self, extra_post_data={}): + def _login_user_and_profile(self, extra_post_data=None): + extra_post_data = {} if extra_post_data is None else extra_post_data profile = self._create_user_profile(extra_post_data) self.user = profile.user self.assertTrue( @@ -127,9 +183,10 @@ def _login_user_and_profile(self, extra_post_data={}): username=self.user.username, password=self.profile_data["password1"] ) ) - self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": f"Token {self.user.auth_token}"} - def _org_create(self, org_data={}): + def _org_create(self, org_data=None): + org_data = {} if org_data is None else org_data view = OrganizationProfileViewSet.as_view({"get": "list", "post": "create"}) request = self.factory.get("/", **self.extra) response = view(request) @@ -155,10 +212,11 @@ def _org_create(self, org_data={}): ) response = view(request) self.assertEqual(response.status_code, 201) - data["url"] = "http://testserver/api/v1/orgs/%s" % data["org"] - data["user"] = "http://testserver/api/v1/users/%s" % data["org"] + data["url"] = f"http://testserver/api/v1/orgs/{data['org']}" + data["user"] = f"http://testserver/api/v1/users/{data['org']}" data["creator"] = "http://testserver/api/v1/users/bob" self.assertDictContainsSubset(data, response.data) + # pylint: disable=attribute-defined-outside-init self.company_data = response.data self.organization = OrganizationProfile.objects.get(user__username=data["org"]) @@ -183,17 +241,18 @@ def _publish_form_with_hxl_support(self): "fixtures", "hxl_example", "instances", - "instance_%s.xml" % x, + f"instance_{x}.xml", ) self._make_submission(path) - def _project_create(self, project_data={}, merge=True): + def _project_create(self, project_data=None, merge=True): + project_data = {} if project_data is None else project_data view = ProjectViewSet.as_view({"post": "create"}) if merge: data = { "name": "demo", - "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "owner": f"http://testserver/api/v1/users/{self.user.username}", "metadata": { "description": "Some description", "location": "Naivasha, Kenya", @@ -210,20 +269,23 @@ def _project_create(self, project_data={}, merge=True): ) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 201) + # pylint: disable=attribute-defined-outside-init self.project = Project.objects.filter(name=data["name"], created_by=self.user)[ 0 ] - data["url"] = "http://testserver/api/v1/projects/%s" % self.project.pk + data["url"] = f"http://testserver/api/v1/projects/{self.project.pk}" self.assertDictContainsSubset(data, response.data) request.user = self.user + # pylint: disable=attribute-defined-outside-init self.project_data = ProjectSerializer( self.project, context={"request": request} ).data def _publish_xls_form_to_project( - self, publish_data={}, merge=True, public=False, xlsform_path=None + self, publish_data=None, merge=True, public=False, xlsform_path=None ): + publish_data = {} if publish_data is None else publish_data if not hasattr(self, "project"): self._project_create() elif self.project.created_by != self.user: @@ -234,8 +296,10 @@ def _publish_xls_form_to_project( project_id = self.project.pk if merge: data = { - "owner": "http://testserver/api/v1/users/%s" - % self.project.organization.username, + "owner": ( + "http://testserver/api/v1/users/" + f"{self.project.organization.username}" + ), "public": False, "public_data": False, "description": "transportation_2011_07_25", @@ -267,30 +331,17 @@ def _publish_xls_form_to_project( request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=project_id) self.assertEqual(response.status_code, 201) + # pylint: disable=attribute-defined-outside-init self.xform = XForm.objects.all().order_by("pk").reverse()[0] - data.update( - {"url": "http://testserver/api/v1/forms/%s" % (self.xform.pk)} - ) + data.update({"url": f"http://testserver/api/v1/forms/{self.xform.pk}"}) # Input was a private so change to public if project public if public: data["public_data"] = data["public"] = True + # pylint: disable=attribute-defined-outside-init self.form_data = response.data - def _add_uuid_to_submission_xml(self, path, xform): - tmp_file = NamedTemporaryFile(delete=False, mode="w") - split_xml = None - - with open(path, encoding="utf-8") as _file: - split_xml = re.split(r"()", _file.read()) - - split_xml[1:1] = ["%s" % xform.uuid] - tmp_file.write("".join(split_xml)) - path = tmp_file.name - tmp_file.close() - - return path - + # pylint: disable=too-many-arguments,too-many-locals,unused-argument def _make_submission( self, path, @@ -311,25 +362,26 @@ def _make_submission( tmp_file = None if add_uuid: - path = self._add_uuid_to_submission_xml(path, self.xform) + path = add_uuid_to_submission_xml(path, self.xform) with open(path, encoding="utf-8") as f: post_data = {"xml_submission_file": f} if media_file is not None: if isinstance(media_file, list): - for c in range(len(media_file)): - post_data["media_file_{}".format(c)] = media_file[c] + for position, _value in enumerate(media_file): + post_data[f"media_file_{position}"] = media_file[position] else: post_data["media_file"] = media_file if username is None: username = self.user.username - url_prefix = "%s/" % username if username else "" - url = "/%ssubmission" % url_prefix + url_prefix = f"{username if username else ''}/" + url = f"/{url_prefix}submission" request = self.factory.post(url, post_data) request.user = authenticate(username=auth.username, password=auth.password) + # pylint: disable=attribute-defined-outside-init self.response = submission(request, username=username) if auth and self.response.status_code == 401: @@ -409,6 +461,7 @@ def _submit_transport_instance_w_attachment( ) attachment = Attachment.objects.all().reverse()[0] + # pylint: disable=attribute-defined-outside-init self.attachment = attachment def _post_metadata(self, data, test=True): @@ -422,6 +475,7 @@ def _post_metadata(self, data, test=True): self.assertEqual(response.status_code, 201, response.data) another_count = MetaData.objects.count() self.assertEqual(another_count, count + 1) + # pylint: disable=attribute-defined-outside-init self.metadata = MetaData.objects.get(pk=response.data["id"]) self.metadata_data = response.data @@ -459,32 +513,33 @@ def _create_dataview(self, data=None, project=None, xform=None): if not data: data = { "name": "My DataView", - "xform": "http://testserver/api/v1/forms/%s" % xform.pk, - "project": "http://testserver/api/v1/projects/%s" % project.pk, + "xform": f"http://testserver/api/v1/forms/{xform.pk}", + "project": f"http://testserver/api/v1/projects/{project.pk}", "columns": '["name", "age", "gender"]', - "query": '[{"column":"age","filter":">","value":"20"},' - '{"column":"age","filter":"<","value":"50"}]', + "query": ( + '[{"column":"age","filter":">","value":"20"},' + '{"column":"age","filter":"<","value":"50"}]' + ), } - request = self.factory.post("/", data=data, **self.extra) - response = view(request) - self.assertEquals(response.status_code, 201) + self.assertEqual(response.status_code, 201) # load the created dataview + # pylint: disable=attribute-defined-outside-init self.data_view = DataView.objects.filter(xform=xform, project=project).last() - self.assertEquals(response.data["name"], data["name"]) - self.assertEquals(response.data["xform"], data["xform"]) - self.assertEquals(response.data["project"], data["project"]) - self.assertEquals(response.data["columns"], json.loads(data["columns"])) - self.assertEquals( + self.assertEqual(response.data["name"], data["name"]) + self.assertEqual(response.data["xform"], data["xform"]) + self.assertEqual(response.data["project"], data["project"]) + self.assertEqual(response.data["columns"], json.loads(data["columns"])) + self.assertEqual( response.data["query"], json.loads(data["query"]) if "query" in data else {} ) - self.assertEquals( + self.assertEqual( response.data["url"], - "http://testserver/api/v1/dataviews/%s" % self.data_view.pk, + f"http://testserver/api/v1/dataviews/{self.data_view.pk}", ) def _create_widget(self, data=None, group_by=""): @@ -493,7 +548,7 @@ def _create_widget(self, data=None, group_by=""): if not data: data = { "title": "Widget that", - "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "content_object": f"http://testserver/api/v1/forms/{self.xform.pk}", "description": "Test widget", "aggregation": "Sum", "widget_type": "charts", @@ -509,23 +564,24 @@ def _create_widget(self, data=None, group_by=""): ) response = view(request) - self.assertEquals(response.status_code, 201) - self.assertEquals(count + 1, Widget.objects.all().count()) + self.assertEqual(response.status_code, 201) + self.assertEqual(count + 1, Widget.objects.all().count()) + # pylint: disable=attribute-defined-outside-init self.widget = Widget.objects.all().order_by("pk").reverse()[0] - self.assertEquals(response.data["id"], self.widget.id) - self.assertEquals(response.data["title"], data.get("title")) - self.assertEquals(response.data["content_object"], data["content_object"]) - self.assertEquals(response.data["widget_type"], data["widget_type"]) - self.assertEquals(response.data["view_type"], data["view_type"]) - self.assertEquals(response.data["column"], data["column"]) - self.assertEquals(response.data["description"], data.get("description")) - self.assertEquals(response.data["group_by"], data.get("group_by")) - self.assertEquals(response.data["aggregation"], data.get("aggregation")) - self.assertEquals(response.data["order"], self.widget.order) - self.assertEquals(response.data["data"], []) - self.assertEquals(response.data["metadata"], data.get("metadata", {})) + self.assertEqual(response.data["id"], self.widget.id) + self.assertEqual(response.data["title"], data.get("title")) + self.assertEqual(response.data["content_object"], data["content_object"]) + self.assertEqual(response.data["widget_type"], data["widget_type"]) + self.assertEqual(response.data["view_type"], data["view_type"]) + self.assertEqual(response.data["column"], data["column"]) + self.assertEqual(response.data["description"], data.get("description")) + self.assertEqual(response.data["group_by"], data.get("group_by")) + self.assertEqual(response.data["aggregation"], data.get("aggregation")) + self.assertEqual(response.data["order"], self.widget.order) + self.assertEqual(response.data["data"], []) + self.assertEqual(response.data["metadata"], data.get("metadata", {})) def _team_create(self): self._org_create() @@ -538,21 +594,25 @@ def _team_create(self): ) response = view(request) self.assertEqual(response.status_code, 201) + # pylint: disable=attribute-defined-outside-init self.owner_team = Team.objects.get( organization=self.organization.user, - name="%s#Owners" % (self.organization.user.username), + name=f"{self.organization.user.username}#Owners", ) team = Team.objects.get( organization=self.organization.user, - name="%s#%s" % (self.organization.user.username, data["name"]), + name=f"{self.organization.user.username}#{data['name']}", ) - data["url"] = "http://testserver/api/v1/teams/%s" % team.pk + data["url"] = f"http://testserver/api/v1/teams/{team.pk}" data["teamid"] = team.id self.assertDictContainsSubset(data, response.data) self.team_data = response.data self.team = team def is_sorted_desc(self, s): + """ + Returns True if a list is sorted in descending order. + """ if len(s) in [0, 1]: return True if s[0] >= s[1]: @@ -560,19 +620,23 @@ def is_sorted_desc(self, s): return False def is_sorted_asc(self, s): + """ + Returns True if a list is sorted in ascending order. + """ if len(s) in [0, 1]: return True if s[0] <= s[1]: return self.is_sorted_asc(s[1:]) return False - def _get_request_session_with_auth(self, view, auth): + def _get_request_session_with_auth(self, view, auth, extra=None): request = self.factory.head("/") response = view(request) self.assertTrue(response.has_header("WWW-Authenticate")) self.assertTrue(response["WWW-Authenticate"].startswith("Digest ")) self.assertIn("nonce=", response["WWW-Authenticate"]) - request = self.factory.get("/") + extra = {} if extra is None else extra + request = self.factory.get("/", **extra) request.META.update(auth(request.META, response)) request.session = self.client.session diff --git a/onadata/apps/api/tests/viewsets/test_attachment_viewset.py b/onadata/apps/api/tests/viewsets/test_attachment_viewset.py index 2ef1815985..4026d6a500 100644 --- a/onadata/apps/api/tests/viewsets/test_attachment_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_attachment_viewset.py @@ -1,11 +1,9 @@ import os -from past.builtins import basestring from django.utils import timezone -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.viewsets.attachment_viewset import AttachmentViewSet from onadata.apps.logger.import_tools import django_file from onadata.apps.logger.models.attachment import Attachment @@ -18,22 +16,15 @@ def attachment_url(attachment, suffix=None): path = get_attachment_url(attachment, suffix) - return u'http://testserver{}'.format(path) + return "http://testserver{}".format(path) class TestAttachmentViewSet(TestAbstractViewSet): - def setUp(self): super(TestAttachmentViewSet, self).setUp() - self.retrieve_view = AttachmentViewSet.as_view({ - 'get': 'retrieve' - }) - self.list_view = AttachmentViewSet.as_view({ - 'get': 'list' - }) - self.count_view = AttachmentViewSet.as_view({ - 'get': 'count' - }) + self.retrieve_view = AttachmentViewSet.as_view({"get": "retrieve"}) + self.list_view = AttachmentViewSet.as_view({"get": "list"}) + self.count_view = AttachmentViewSet.as_view({"get": "count"}) self._publish_xls_form_to_project() @@ -43,36 +34,36 @@ def test_retrieve_view(self): pk = self.attachment.pk data = { - 'url': 'http://testserver/api/v1/media/%s' % pk, - 'field_xpath': 'image1', - 'download_url': attachment_url(self.attachment), - 'small_download_url': attachment_url(self.attachment, 'small'), - 'medium_download_url': attachment_url(self.attachment, 'medium'), - 'id': pk, - 'xform': self.xform.pk, - 'instance': self.attachment.instance.pk, - 'mimetype': self.attachment.mimetype, - 'filename': self.attachment.media_file.name + "url": "http://testserver/api/v1/media/%s" % pk, + "field_xpath": "image1", + "download_url": attachment_url(self.attachment), + "small_download_url": attachment_url(self.attachment, "small"), + "medium_download_url": attachment_url(self.attachment, "medium"), + "id": pk, + "xform": self.xform.pk, + "instance": self.attachment.instance.pk, + "mimetype": self.attachment.mimetype, + "filename": self.attachment.media_file.name, } - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.retrieve_view(request, pk=pk) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, dict)) self.assertEqual(response.data, data) # file download - filename = data['filename'] - ext = filename[filename.rindex('.') + 1:] - request = self.factory.get('/', **self.extra) + filename = data["filename"] + ext = filename[filename.rindex(".") + 1 :] + request = self.factory.get("/", **self.extra) response = self.retrieve_view(request, pk=pk, format=ext) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - self.assertEqual(response.content_type, 'image/jpeg') + self.assertEqual(response.content_type, "image/jpeg") self.attachment.instance.xform.deleted_at = timezone.now() self.attachment.instance.xform.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.retrieve_view(request, pk=pk) self.assertEqual(response.status_code, 404) @@ -83,41 +74,46 @@ def test_attachment_pagination(self): self._submit_transport_instance_w_attachment() self.assertEqual(self.response.status_code, 201) filename = "1335783522564.JPG" - path = os.path.join(self.main_directory, 'fixtures', 'transportation', - 'instances', self.surveys[0], filename) - media_file = django_file(path, 'image2', 'image/jpeg') + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + filename, + ) + media_file = django_file(path, "image2", "image/jpeg") Attachment.objects.create( instance=self.xform.instances.first(), - mimetype='image/jpeg', - extension='JPG', + mimetype="image/jpeg", + extension="JPG", name=filename, - media_file=media_file) + media_file=media_file, + ) # not using pagination params - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 2) # valid page and page_size - request = self.factory.get( - '/', data={"page": 1, "page_size": 1}, **self.extra) + request = self.factory.get("/", data={"page": 1, "page_size": 1}, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) # invalid page type - request = self.factory.get('/', data={"page": "invalid"}, **self.extra) + request = self.factory.get("/", data={"page": "invalid"}, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 404) # invalid page size type - request = self.factory.get('/', data={"page_size": "invalid"}, - **self.extra) + request = self.factory.get("/", data={"page_size": "invalid"}, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) @@ -125,14 +121,13 @@ def test_attachment_pagination(self): # invalid page and page_size types request = self.factory.get( - '/', data={"page": "invalid", "page_size": "invalid"}, - **self.extra) + "/", data={"page": "invalid", "page_size": "invalid"}, **self.extra + ) response = self.list_view(request) self.assertEqual(response.status_code, 404) # invalid page size - request = self.factory.get( - '/', data={"page": 4, "page_size": 1}, **self.extra) + request = self.factory.get("/", data={"page": 4, "page_size": 1}, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 404) @@ -143,16 +138,16 @@ def test_retrieve_and_list_views_with_anonymous_user(self): pk = self.attachment.pk xform_id = self.attachment.instance.xform.id - request = self.factory.get('/') + request = self.factory.get("/") response = self.retrieve_view(request, pk=pk) self.assertEqual(response.status_code, 404) - request = self.factory.get('/') + request = self.factory.get("/") response = self.list_view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) - request = self.factory.get('/', data={"xform": xform_id}) + request = self.factory.get("/", data={"xform": xform_id}) response = self.list_view(request) self.assertEqual(response.status_code, 404) @@ -160,24 +155,24 @@ def test_retrieve_and_list_views_with_anonymous_user(self): xform.shared_data = True xform.save() - request = self.factory.get('/') + request = self.factory.get("/") response = self.retrieve_view(request, pk=pk) self.assertEqual(response.status_code, 200) - request = self.factory.get('/') + request = self.factory.get("/") response = self.list_view(request) self.assertEqual(response.status_code, 200) - request = self.factory.get('/', data={"xform": xform_id}) + request = self.factory.get("/", data={"xform": xform_id}) response = self.list_view(request) self.assertEqual(response.status_code, 200) def test_list_view(self): self._submit_transport_instance_w_attachment() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) @@ -186,7 +181,7 @@ def test_list_view(self): self.attachment.instance.deleted_at = timezone.now() self.attachment.instance.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.list_view(request) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 0) @@ -194,16 +189,16 @@ def test_list_view(self): def test_data_list_with_xform_in_delete_async(self): self._submit_transport_instance_w_attachment() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) initial_count = len(response.data) self.xform.deleted_at = timezone.now() self.xform.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), initial_count - 1) @@ -211,204 +206,215 @@ def test_data_list_with_xform_in_delete_async(self): def test_list_view_filter_by_xform(self): self._submit_transport_instance_w_attachment() - data = { - 'xform': self.xform.pk - } - request = self.factory.get('/', data, **self.extra) + data = {"xform": self.xform.pk} + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) - data['xform'] = 10000000 - request = self.factory.get('/', data, **self.extra) + data["xform"] = 10000000 + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 404) - data['xform'] = 'lol' - request = self.factory.get('/', data, **self.extra) + data["xform"] = "lol" + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) def test_list_view_filter_by_instance(self): self._submit_transport_instance_w_attachment() - data = { - 'instance': self.attachment.instance.pk - } - request = self.factory.get('/', data, **self.extra) + data = {"instance": self.attachment.instance.pk} + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) - data['instance'] = 10000000 - request = self.factory.get('/', data, **self.extra) + data["instance"] = 10000000 + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 404) - data['instance'] = 'lol' - request = self.factory.get('/', data, **self.extra) + data["instance"] = "lol" + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) def test_list_view_filter_by_attachment_type(self): self._submit_transport_instance_w_attachment() filename = "1335783522564.JPG" - path = os.path.join(self.main_directory, 'fixtures', 'transportation', - 'instances', self.surveys[0], filename) - media_file = django_file(path, 'video2', 'image/jpeg') + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + filename, + ) + media_file = django_file(path, "video2", "image/jpeg") # test geojson attachments - geojson_filename = 'sample.geojson' - geojson_path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', - self.surveys[0], geojson_filename) - geojson_media_file = django_file( - geojson_path, 'store_gps', 'image/jpeg') + geojson_filename = "sample.geojson" + geojson_path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + geojson_filename, + ) + geojson_media_file = django_file(geojson_path, "store_gps", "image/jpeg") Attachment.objects.create( instance=self.xform.instances.first(), - mimetype='video/mp4', - extension='MP4', + mimetype="video/mp4", + extension="MP4", name=filename, - media_file=media_file) + media_file=media_file, + ) Attachment.objects.create( instance=self.xform.instances.first(), - mimetype='application/pdf', - extension='PDF', + mimetype="application/pdf", + extension="PDF", name=filename, - media_file=media_file) + media_file=media_file, + ) Attachment.objects.create( instance=self.xform.instances.first(), - mimetype='text/plain', - extension='TXT', + mimetype="text/plain", + extension="TXT", name=filename, - media_file=media_file) + media_file=media_file, + ) Attachment.objects.create( instance=self.xform.instances.first(), - mimetype='audio/mp3', - extension='MP3', + mimetype="audio/mp3", + extension="MP3", name=filename, - media_file=media_file) + media_file=media_file, + ) Attachment.objects.create( instance=self.xform.instances.first(), - mimetype='application/geo+json', - extension='GEOJSON', + mimetype="application/geo+json", + extension="GEOJSON", name=geojson_filename, - media_file=geojson_media_file) + media_file=geojson_media_file, + ) data = {} - request = self.factory.get('/', data, **self.extra) + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 6) # Apply image Filter - data['type'] = 'image' - request = self.factory.get('/', data, **self.extra) + data["type"] = "image" + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) - self.assertEqual(response.data[0]["mimetype"], 'image/jpeg') + self.assertEqual(response.data[0]["mimetype"], "image/jpeg") # Apply audio filter - data['type'] = 'audio' - request = self.factory.get('/', data, **self.extra) + data["type"] = "audio" + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) - self.assertEqual(response.data[0]["mimetype"], 'audio/mp3') + self.assertEqual(response.data[0]["mimetype"], "audio/mp3") # Apply video filter - data['type'] = 'video' - request = self.factory.get('/', data, **self.extra) + data["type"] = "video" + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) - self.assertEqual(response.data[0]["mimetype"], 'video/mp4') + self.assertEqual(response.data[0]["mimetype"], "video/mp4") # Apply file filter - data['type'] = 'document' - request = self.factory.get('/', data, **self.extra) + data["type"] = "document" + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 3) - self.assertEqual(response.data[0]["mimetype"], 'application/pdf') - self.assertEqual(response.data[1]["mimetype"], 'text/plain') - self.assertEqual(response.data[2]["mimetype"], 'application/geo+json') + self.assertEqual(response.data[0]["mimetype"], "application/pdf") + self.assertEqual(response.data[1]["mimetype"], "text/plain") + self.assertEqual(response.data[2]["mimetype"], "application/geo+json") def test_direct_image_link(self): self._submit_transport_instance_w_attachment() - data = { - 'filename': self.attachment.media_file.name - } - request = self.factory.get('/', data, **self.extra) + data = {"filename": self.attachment.media_file.name} + request = self.factory.get("/", data, **self.extra) response = self.retrieve_view(request, pk=self.attachment.pk) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - self.assertTrue(isinstance(response.data, basestring)) + self.assertTrue(isinstance(response.data, str)) self.assertEqual(response.data, attachment_url(self.attachment)) - data['filename'] = 10000000 - request = self.factory.get('/', data, **self.extra) + data["filename"] = 10000000 + request = self.factory.get("/", data, **self.extra) response = self.retrieve_view(request, pk=self.attachment.instance.pk) self.assertEqual(response.status_code, 404) - data['filename'] = 'lol' - request = self.factory.get('/', data, **self.extra) + data["filename"] = "lol" + request = self.factory.get("/", data, **self.extra) response = self.retrieve_view(request, pk=self.attachment.instance.pk) self.assertEqual(response.status_code, 404) def test_direct_image_link_uppercase(self): self._submit_transport_instance_w_attachment() filename = "1335783522564.JPG" - path = os.path.join(self.main_directory, 'fixtures', 'transportation', - 'instances', self.surveys[0], filename) - self.attachment.media_file = django_file(path, 'image2', 'image/jpeg') + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + filename, + ) + self.attachment.media_file = django_file(path, "image2", "image/jpeg") self.attachment.name = filename self.attachment.save() filename = self.attachment.media_file.name file_base, file_extension = os.path.splitext(filename) - data = { - 'filename': file_base + file_extension.upper() - } - request = self.factory.get('/', data, **self.extra) + data = {"filename": file_base + file_extension.upper()} + request = self.factory.get("/", data, **self.extra) response = self.retrieve_view(request, pk=self.attachment.pk) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - self.assertTrue(isinstance(response.data, basestring)) + self.assertTrue(isinstance(response.data, str)) self.assertEqual(response.data, attachment_url(self.attachment)) def test_total_count(self): self._submit_transport_instance_w_attachment() xform_id = self.attachment.instance.xform.id - request = self.factory.get( - '/count', data={"xform": xform_id}, **self.extra) + request = self.factory.get("/count", data={"xform": xform_id}, **self.extra) response = self.count_view(request) - self.assertEqual(response.data['count'], 1) + self.assertEqual(response.data["count"], 1) def test_returned_attachments_is_based_on_form_permissions(self): # Create a form and make submissions with attachments self._submit_transport_instance_w_attachment() formid = self.xform.pk - request = self.factory.get( - '/', data={"xform": formid}, **self.extra) + request = self.factory.get("/", data={"xform": formid}, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) @@ -419,13 +425,11 @@ def test_returned_attachments_is_based_on_form_permissions(self): MetaData.xform_meta_permission(self.xform, data_value=data_value) ShareXForm(self.xform, user_dave.username, EditorRole.name) - auth_extra = { - 'HTTP_AUTHORIZATION': f'Token {user_dave.auth_token.key}' - } + auth_extra = {"HTTP_AUTHORIZATION": f"Token {user_dave.auth_token.key}"} # Dave user should not be able to view attachments for # submissions which they did not submit - request = self.factory.get('/', data={"xform": formid}, **auth_extra) + request = self.factory.get("/", data={"xform": formid}, **auth_extra) response = self.list_view(request) self.assertEqual(response.status_code, 200) # Ensure no submissions are returned for the User diff --git a/onadata/apps/api/tests/viewsets/test_charts_viewset.py b/onadata/apps/api/tests/viewsets/test_charts_viewset.py index 6a50d78273..e340e5c446 100644 --- a/onadata/apps/api/tests/viewsets/test_charts_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_charts_viewset.py @@ -94,7 +94,7 @@ def test_correct_merged_dataset_data_for_charts(self): 'name': 'Merged Dataset', 'project': - "http://testserver/api/v1/projects/%s" % self.project.pk, + f"http://testserver/api/v1/projects/{self.project.pk}", } # anonymous user request = self.factory.post('/', data=data) diff --git a/onadata/apps/api/tests/viewsets/test_connect_viewset.py b/onadata/apps/api/tests/viewsets/test_connect_viewset.py index 9b23639510..ae3053e51d 100644 --- a/onadata/apps/api/tests/viewsets/test_connect_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_connect_viewset.py @@ -1,3 +1,7 @@ +# -*- coding=utf-8 -*- +""" +Test /user API endpoint +""" from datetime import datetime from datetime import timedelta @@ -16,11 +20,9 @@ from onadata.apps.api.models.temp_token import TempToken from onadata.apps.api.models.odk_token import ODKToken -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.viewsets.connect_viewset import ConnectViewSet -from onadata.libs.serializers.password_reset_serializer import \ - default_token_generator +from onadata.libs.serializers.password_reset_serializer import default_token_generator from onadata.apps.api.viewsets.project_viewset import ProjectViewSet from onadata.libs.authentication import DigestAuthentication from onadata.libs.serializers.project_serializer import ProjectSerializer @@ -28,43 +30,48 @@ class TestConnectViewSet(TestAbstractViewSet): - def setUp(self): super(self.__class__, self).setUp() - self.view = ConnectViewSet.as_view({ - "get": "list", - "post": "reset", - "delete": "expire", - }) + self.view = ConnectViewSet.as_view( + { + "get": "list", + "post": "reset", + "delete": "expire", + } + ) self.data = { - 'url': 'http://testserver/api/v1/profiles/bob', - 'username': u'bob', - 'name': u'Bob', - 'email': u'bob@columbia.edu', - 'city': u'Bobville', - 'country': u'US', - 'organization': u'Bob Inc.', - 'website': u'bob.com', - 'twitter': u'boberama', - 'gravatar': self.user.profile.gravatar, - 'require_auth': False, - 'user': 'http://testserver/api/v1/users/bob', - 'api_token': self.user.auth_token.key, + "url": "http://testserver/api/v1/profiles/bob", + "username": "bob", + "name": "Bob", + "email": "bob@columbia.edu", + "city": "Bobville", + "country": "US", + "organization": "Bob Inc.", + "website": "bob.com", + "twitter": "boberama", + "gravatar": self.user.profile.gravatar, + "require_auth": False, + "user": "http://testserver/api/v1/users/bob", + "api_token": self.user.auth_token.key, } def test_generate_auth_token(self): - self.view = ConnectViewSet.as_view({ - "post": "create", - }) + self.view = ConnectViewSet.as_view( + { + "post": "create", + } + ) request = self.factory.post("/", **self.extra) request.session = self.client.session response = self.view(request) self.assertEqual(response.status_code, 201) def test_regenerate_auth_token(self): - self.view = ConnectViewSet.as_view({ - "get": "regenerate_auth_token", - }) + self.view = ConnectViewSet.as_view( + { + "get": "regenerate_auth_token", + } + ) prev_token = self.user.auth_token request = self.factory.get("/", **self.extra) response = self.view(request) @@ -72,62 +79,64 @@ def test_regenerate_auth_token(self): new_token = Token.objects.get(user=self.user) self.assertNotEqual(prev_token, new_token) - self.view = ConnectViewSet.as_view({ - "get": "list", - }) - self.extra = {'HTTP_AUTHORIZATION': 'Token %s' % new_token} - request = self.factory.get('/', **self.extra) + self.view = ConnectViewSet.as_view( + { + "get": "list", + } + ) + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % new_token} + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) self.assertEqual(response.status_code, 200) - self.extra = {'HTTP_AUTHORIZATION': 'Token invalidtoken'} - request = self.factory.get('/', **self.extra) + self.extra = {"HTTP_AUTHORIZATION": "Token invalidtoken"} + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response['www-authenticate'], "Token") + self.assertEqual(response["www-authenticate"], "Token") def test_get_profile(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) - temp_token = TempToken.objects.get(user__username='bob') - self.data['temp_token'] = temp_token.key + temp_token = TempToken.objects.get(user__username="bob") + self.data["temp_token"] = temp_token.key self.assertEqual(response.status_code, 200) self.assertEqual(dict(response.data), self.data) def test_using_valid_temp_token(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) - temp_token = response.data['temp_token'] + temp_token = response.data["temp_token"] - self.extra = {'HTTP_AUTHORIZATION': 'TempToken %s' % temp_token} - request = self.factory.get('/', **self.extra) + self.extra = {"HTTP_AUTHORIZATION": "TempToken %s" % temp_token} + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) self.assertEqual(response.status_code, 200) - self.assertEqual(temp_token, response.data['temp_token']) + self.assertEqual(temp_token, response.data["temp_token"]) def test_using_invalid_temp_token(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) - temp_token = 'abcdefghijklmopqrstuvwxyz' + temp_token = "abcdefghijklmopqrstuvwxyz" - self.extra = {'HTTP_AUTHORIZATION': 'TempToken %s' % temp_token} - request = self.factory.get('/', **self.extra) + self.extra = {"HTTP_AUTHORIZATION": "TempToken %s" % temp_token} + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response.data['detail'], 'Invalid token') - self.assertEqual(response['www-authenticate'], "TempToken") + self.assertEqual(response.data["detail"], "Invalid token") + self.assertEqual(response["www-authenticate"], "TempToken") def test_using_expired_temp_token(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) - temp_token = response.data['temp_token'] + temp_token = response.data["temp_token"] temp_token_obj = TempToken.objects.get(key=temp_token) day = timedelta(seconds=settings.DEFAULT_TEMP_TOKEN_EXPIRY_TIME) @@ -136,67 +145,65 @@ def test_using_expired_temp_token(self): temp_token_obj.created = yesterday temp_token_obj.save() - self.extra = {'HTTP_AUTHORIZATION': 'TempToken %s' % - temp_token_obj.key} - request = self.factory.get('/', **self.extra) + self.extra = {"HTTP_AUTHORIZATION": "TempToken %s" % temp_token_obj.key} + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) - self.assertEqual(response.data['detail'], 'Token expired') + self.assertEqual(response.data["detail"], "Token expired") def test_expire_temp_token_using_expire_endpoint(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) - temp_token = response.data['temp_token'] + temp_token = response.data["temp_token"] # expire temporary token - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) request.session = self.client.session response = self.view(request) self.assertEqual(response.status_code, 204) # try to expire temporary token for the second time - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) request.session = self.client.session response = self.view(request) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data['detail'], 'Temporary token not found!') + self.assertEqual(response.data["detail"], "Temporary token not found!") # try to login with deleted temporary token - self.extra = {'HTTP_AUTHORIZATION': 'TempToken %s' % temp_token} - request = self.factory.get('/', **self.extra) + self.extra = {"HTTP_AUTHORIZATION": "TempToken %s" % temp_token} + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response.data['detail'], - 'Invalid token') - self.assertEqual(response['www-authenticate'], "TempToken") + self.assertEqual(response.data["detail"], "Invalid token") + self.assertEqual(response["www-authenticate"], "TempToken") def test_get_starred_projects(self): self._project_create() # add star as bob - view = ProjectViewSet.as_view({ - 'get': 'star', - 'post': 'star' - }) - request = self.factory.post('/', **self.extra) + view = ProjectViewSet.as_view({"get": "star", "post": "star"}) + request = self.factory.post("/", **self.extra) response = view(request, pk=self.project.pk) # get starred projects - view = ConnectViewSet.as_view({ - 'get': 'starred', - }) - request = self.factory.get('/', **self.extra) + view = ConnectViewSet.as_view( + { + "get": "starred", + } + ) + request = self.factory.get("/", **self.extra) response = view(request, user=self.user) self.assertEqual(response.status_code, 200) self.project.refresh_from_db() request.user = self.user self.project_data = ProjectSerializer( - self.project, context={'request': request}).data - del self.project_data['date_modified'] - del response.data[0]['date_modified'] + self.project, context={"request": request} + ).data + del self.project_data["date_modified"] + del response.data[0]["date_modified"] self.assertEqual(len(response.data), 1) self.assertDictEqual(dict(response.data[0]), dict(self.project_data)) @@ -205,81 +212,85 @@ def test_user_list_with_digest(self): cache.clear() view = ConnectViewSet.as_view( - {'get': 'list'}, - authentication_classes=(DigestAuthentication,)) - request = self.factory.head('/') + {"get": "list"}, authentication_classes=(DigestAuthentication,) + ) + request = self.factory.head("/") - auth = DigestAuth('bob', 'bob') + auth = DigestAuth("bob", "bob") response = view(request) - self.assertTrue(response.has_header('WWW-Authenticate')) - self.assertTrue(response['WWW-Authenticate'].startswith('Digest ')) - self.assertIn('nonce=', response['WWW-Authenticate']) - request = self.factory.get('/') + self.assertTrue(response.has_header("WWW-Authenticate")) + self.assertTrue(response["WWW-Authenticate"].startswith("Digest ")) + self.assertIn("nonce=", response["WWW-Authenticate"]) + request = self.factory.get("/") request.META.update(auth(request.META, response)) request.session = self.client.session response = view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response.data['detail'], - u"Invalid username/password. For security reasons, " - u"after 9 more failed login attempts you'll " - u"have to wait 30 minutes before trying again.") - auth = DigestAuth('bob', 'bobbob') + self.assertEqual( + response.data["detail"], + "Invalid username/password. For security reasons, " + "after 9 more failed login attempts you'll " + "have to wait 30 minutes before trying again.", + ) + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) request.session = self.client.session response = view(request) - temp_token = TempToken.objects.get(user__username='bob') - self.data['temp_token'] = temp_token.key + temp_token = TempToken.objects.get(user__username="bob") + self.data["temp_token"] = temp_token.key self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.data) def test_user_list_with_basic_and_digest(self): view = ConnectViewSet.as_view( - {'get': 'list'}, + {"get": "list"}, authentication_classes=( DigestAuthentication, - authentication.BasicAuthentication - )) - request = self.factory.get('/') - auth = BasicAuth('bob', 'bob') + authentication.BasicAuthentication, + ), + ) + request = self.factory.get("/") + auth = BasicAuth("bob", "bob") request.META.update(auth(request.META)) request.session = self.client.session response = view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response.data['detail'], - u"Invalid username/password.") - auth = BasicAuth('bob', 'bobbob') + self.assertEqual(response.data["detail"], "Invalid username/password.") + auth = BasicAuth("bob", "bobbob") request.META.update(auth(request.META)) request.session = self.client.session response = view(request) - temp_token = TempToken.objects.get(user__username='bob') - self.data['temp_token'] = temp_token.key + temp_token = TempToken.objects.get(user__username="bob") + self.data["temp_token"] = temp_token.key self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.data) - @patch('onadata.libs.serializers.password_reset_serializer.send_mail') + @patch("onadata.libs.serializers.password_reset_serializer.send_mail") def test_request_reset_password(self, mock_send_mail): - data = {'email': self.user.email, - 'reset_url': u'http://testdomain.com/reset_form'} - request = self.factory.post('/', data=data) + data = { + "email": self.user.email, + "reset_url": "http://testdomain.com/reset_form", + } + request = self.factory.post("/", data=data) response = self.view(request) self.assertEqual(response.status_code, 204, response.data) self.assertTrue(mock_send_mail.called) - data['email_subject'] = 'X' * 100 - request = self.factory.post('/', data=data) + data["email_subject"] = "X" * 100 + request = self.factory.post("/", data=data) response = self.view(request) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data['email_subject'][0], - u'Ensure this field has no more than 78 characters.' + response.data["email_subject"][0], + "Ensure this field has no more than 78 characters.", ) mock_send_mail.called = False - request = self.factory.post('/') + request = self.factory.post("/") response = self.view(request) self.assertFalse(mock_send_mail.called) self.assertEqual(response.status_code, 400) @@ -291,23 +302,22 @@ def test_reset_user_password(self): self.user.save() token = default_token_generator.make_token(self.user) new_password = "bobbob1" - data = {'token': token, 'new_password': new_password} + data = {"token": token, "new_password": new_password} # missing uid, should fail - request = self.factory.post('/', data=data) + request = self.factory.post("/", data=data) response = self.view(request) self.assertEqual(response.status_code, 400) - data['uid'] = urlsafe_base64_encode( - force_bytes(self.user.pk)) + data["uid"] = urlsafe_base64_encode(force_bytes(self.user.pk)) # with uid, should be successful - request = self.factory.post('/', data=data) + request = self.factory.post("/", data=data) response = self.view(request) self.assertEqual(response.status_code, 200) user = User.objects.get(email=self.user.email) - self.assertEqual(user.username, response.data['username']) + self.assertEqual(user.username, response.data["username"]) self.assertTrue(user.check_password(new_password)) - request = self.factory.post('/', data=data) + request = self.factory.post("/", data=data) response = self.view(request) self.assertEqual(response.status_code, 400) @@ -317,12 +327,10 @@ def test_reset_user_password_with_updated_user_email(self): self.user.last_login = now() self.user.save() new_password = "bobbob1" - uid = urlsafe_base64_encode( - force_bytes(self.user.pk)) + uid = urlsafe_base64_encode(force_bytes(self.user.pk)) mhv = default_token_generator token = mhv.make_token(self.user) - data = {'token': token, 'new_password': new_password, - 'uid': uid} + data = {"token": token, "new_password": new_password, "uid": uid} # check that the token is valid valid_token = mhv.check_token(self.user, token) self.assertTrue(valid_token) @@ -336,18 +344,19 @@ def test_reset_user_password_with_updated_user_email(self): invalid_token = mhv.check_token(self.user, token) self.assertFalse(invalid_token) - request = self.factory.post('/', data=data) + request = self.factory.post("/", data=data) response = self.view(request) self.assertEqual(response.status_code, 400) - self.assertTrue( - 'Invalid token' in response.data['non_field_errors'][0]) + self.assertTrue("Invalid token" in response.data["non_field_errors"][0]) - @patch('onadata.libs.serializers.password_reset_serializer.send_mail') + @patch("onadata.libs.serializers.password_reset_serializer.send_mail") def test_request_reset_password_custom_email_subject(self, mock_send_mail): - data = {'email': self.user.email, - 'reset_url': u'http://testdomain.com/reset_form', - 'email_subject': 'You requested for a reset password'} - request = self.factory.post('/', data=data) + data = { + "email": self.user.email, + "reset_url": "http://testdomain.com/reset_form", + "email_subject": "You requested for a reset password", + } + request = self.factory.post("/", data=data) response = self.view(request) self.assertTrue(mock_send_mail.called) @@ -357,30 +366,32 @@ def test_user_updates_email_wrong_password(self): # Clear cache cache.clear() view = ConnectViewSet.as_view( - {'get': 'list'}, - authentication_classes=(DigestAuthentication,)) + {"get": "list"}, authentication_classes=(DigestAuthentication,) + ) - auth = DigestAuth('bob@columbia.edu', 'bob') + auth = DigestAuth("bob@columbia.edu", "bob") request = self._get_request_session_with_auth(view, auth) response = view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response.data['detail'], - u"Invalid username/password. For security reasons, " - u"after 9 more failed login attempts you'll have to " - u"wait 30 minutes before trying again.") + self.assertEqual( + response.data["detail"], + "Invalid username/password. For security reasons, " + "after 9 more failed login attempts you'll have to " + "wait 30 minutes before trying again.", + ) def test_user_updates_email(self): view = ConnectViewSet.as_view( - {'get': 'list'}, - authentication_classes=(DigestAuthentication,)) + {"get": "list"}, authentication_classes=(DigestAuthentication,) + ) - auth = DigestAuth('bob@columbia.edu', 'bobbob') + auth = DigestAuth("bob@columbia.edu", "bobbob") request = self._get_request_session_with_auth(view, auth) response = view(request) - temp_token = TempToken.objects.get(user__username='bob') - self.data['temp_token'] = temp_token.key + temp_token = TempToken.objects.get(user__username="bob") + self.data["temp_token"] = temp_token.key self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.data) @@ -388,35 +399,35 @@ def test_user_updates_email(self): self.user.save() update_partial_digests(self.user, "bobbob") - auth = DigestAuth('bob2@columbia.edu', 'bobbob') + auth = DigestAuth("bob2@columbia.edu", "bobbob") request = self._get_request_session_with_auth(view, auth) response = view(request) - temp_token = TempToken.objects.get(user__username='bob') - self.data['temp_token'] = temp_token.key - self.data['email'] = 'bob2@columbia.edu' + temp_token = TempToken.objects.get(user__username="bob") + self.data["temp_token"] = temp_token.key + self.data["email"] = "bob2@columbia.edu" self.assertEqual(response.status_code, 200) def test_user_has_no_profile_bug(self): - alice = User.objects.create(username='alice') - alice.set_password('alice') + alice = User.objects.create(username="alice") + alice.set_password("alice") update_partial_digests(alice, "alice") view = ConnectViewSet.as_view( - {'get': 'list'}, - authentication_classes=(DigestAuthentication,)) + {"get": "list"}, authentication_classes=(DigestAuthentication,) + ) - auth = DigestAuth('alice', 'alice') + auth = DigestAuth("alice", "alice") request = self._get_request_session_with_auth(view, auth) response = view(request) self.assertEqual(response.status_code, 200) - @patch('onadata.apps.api.tasks.send_account_lockout_email.apply_async') + @patch("onadata.apps.api.tasks.send_account_lockout_email.apply_async") def test_login_attempts(self, send_account_lockout_email): view = ConnectViewSet.as_view( - {'get': 'list'}, - authentication_classes=(DigestAuthentication,)) - auth = DigestAuth('bob', 'bob') + {"get": "list"}, authentication_classes=(DigestAuthentication,) + ) + auth = DigestAuth("bob", "bob") # clear cache cache.clear() @@ -425,90 +436,90 @@ def test_login_attempts(self, send_account_lockout_email): # first time it creates a cache response = view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response.data['detail'], - u"Invalid username/password. For security reasons, " - u"after 9 more failed login attempts you'll have to " - u"wait 30 minutes before trying again.") - request_ip = request.META.get('REMOTE_ADDR') self.assertEqual( - cache.get(safe_key(f'login_attempts-{request_ip}-bob')), 1) + response.data["detail"], + "Invalid username/password. For security reasons, " + "after 9 more failed login attempts you'll have to " + "wait 30 minutes before trying again.", + ) + request_ip = request.META.get("REMOTE_ADDR") + self.assertEqual(cache.get(safe_key(f"login_attempts-{request_ip}-bob")), 1) # cache value increments with subsequent attempts response = view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response.data['detail'], - u"Invalid username/password. For security reasons, " - u"after 8 more failed login attempts you'll have to " - u"wait 30 minutes before trying again.") self.assertEqual( - cache.get(safe_key(f'login_attempts-{request_ip}-bob')), 2) + response.data["detail"], + "Invalid username/password. For security reasons, " + "after 8 more failed login attempts you'll have to " + "wait 30 minutes before trying again.", + ) + self.assertEqual(cache.get(safe_key(f"login_attempts-{request_ip}-bob")), 2) + request = self._get_request_session_with_auth( + view, auth, extra={"HTTP_X_REAL_IP": "5.6.7.8"} + ) # login attempts are tracked separately for other IPs - request.META.update({'HTTP_X_REAL_IP': '5.6.7.8'}) response = view(request) self.assertEqual(response.status_code, 401) - self.assertEqual( - cache.get(safe_key(f'login_attempts-{request_ip}-bob')), 2) - self.assertEqual( - cache.get(safe_key('login_attempts-5.6.7.8-bob')), 1 - ) + self.assertEqual(cache.get(safe_key(f"login_attempts-{request_ip}-bob")), 2) + self.assertEqual(cache.get(safe_key("login_attempts-5.6.7.8-bob")), 1) # login_attempts doesn't increase with correct login - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request = self._get_request_session_with_auth(view, auth) response = view(request) self.assertEqual(response.status_code, 200) - self.assertEqual( - cache.get(safe_key(f'login_attempts-{request_ip}-bob')), 2) + self.assertEqual(cache.get(safe_key(f"login_attempts-{request_ip}-bob")), 2) # lockout_user cache created upon fifth attempt - auth = DigestAuth('bob', 'bob') + auth = DigestAuth("bob", "bob") request = self._get_request_session_with_auth(view, auth) self.assertFalse(send_account_lockout_email.called) - cache.set(safe_key(f'login_attempts-{request_ip}-bob'), 9) - self.assertIsNone(cache.get(safe_key(f'lockout_ip-{request_ip}-bob'))) + cache.set(safe_key(f"login_attempts-{request_ip}-bob"), 9) + self.assertIsNone(cache.get(safe_key(f"lockout_ip-{request_ip}-bob"))) response = view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response.data['detail'], - u"Locked out. Too many wrong username/password " - u"attempts. Try again in 30 minutes.") self.assertEqual( - cache.get(safe_key(f'login_attempts-{request_ip}-bob')), 10) - self.assertIsNotNone( - cache.get(safe_key(f'lockout_ip-{request_ip}-bob'))) + response.data["detail"], + "Locked out. Too many wrong username/password " + "attempts. Try again in 30 minutes.", + ) + self.assertEqual(cache.get(safe_key(f"login_attempts-{request_ip}-bob")), 10) + self.assertIsNotNone(cache.get(safe_key(f"lockout_ip-{request_ip}-bob"))) lockout = datetime.strptime( - cache.get(safe_key(f'lockout_ip-{request_ip}-bob')), - '%Y-%m-%dT%H:%M:%S') + cache.get(safe_key(f"lockout_ip-{request_ip}-bob")), "%Y-%m-%dT%H:%M:%S" + ) self.assertIsInstance(lockout, datetime) # email sent upon limit being reached with right arguments - subject_path = 'account_lockout/lockout_email_subject.txt' + subject_path = "account_lockout/lockout_email_subject.txt" self.assertTrue(send_account_lockout_email.called) email_subject = render_to_string(subject_path) - self.assertIn( - email_subject, send_account_lockout_email.call_args[1]['args']) - self.assertEqual( - send_account_lockout_email.call_count, 2, "Called twice") + self.assertIn(email_subject, send_account_lockout_email.call_args[1]["args"]) + self.assertEqual(send_account_lockout_email.call_count, 2, "Called twice") # subsequent login fails after lockout even with correct credentials - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request = self._get_request_session_with_auth(view, auth) response = view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response.data['detail'], - u"Locked out. Too many wrong username/password " - u"attempts. Try again in 30 minutes.") + self.assertEqual( + response.data["detail"], + "Locked out. Too many wrong username/password " + "attempts. Try again in 30 minutes.", + ) # Other users on same IP not locked out - alice = User.objects.create(username='alice') - alice.set_password('alice') + alice = User.objects.create(username="alice") + alice.set_password("alice") update_partial_digests(alice, "alice") - auth = DigestAuth('alice', 'alice') + auth = DigestAuth("alice", "alice") request = self._get_request_session_with_auth(view, auth) response = view(request) self.assertEqual(response.status_code, 200) - self.assertEqual(request.META.get('REMOTE_ADDR'), request_ip) + self.assertEqual(request.META.get("REMOTE_ADDR"), request_ip) # clear cache cache.clear() @@ -516,8 +527,8 @@ def test_generate_odk_token(self): """ Test that ODK Tokens can be created """ - view = ConnectViewSet.as_view({'post': 'odk_token'}) - request = self.factory.post('/', **self.extra) + view = ConnectViewSet.as_view({"post": "odk_token"}) + request = self.factory.post("/", **self.extra) request.session = self.client.session response = view(request) self.assertEqual(response.status_code, 201) @@ -527,59 +538,51 @@ def test_regenerate_odk_token(self): Test that ODK Tokens can be regenerated and old tokens are set to Inactive after regeneration """ - view = ConnectViewSet.as_view({'post': 'odk_token'}) - request = self.factory.post('/', **self.extra) + view = ConnectViewSet.as_view({"post": "odk_token"}) + request = self.factory.post("/", **self.extra) request.session = self.client.session response = view(request) self.assertEqual(response.status_code, 201) - old_token = response.data['odk_token'] + old_token = response.data["odk_token"] with self.assertRaises(ODKToken.DoesNotExist): - ODKToken.objects.get( - user=self.user, status=ODKToken.INACTIVE) + ODKToken.objects.get(user=self.user, status=ODKToken.INACTIVE) - request = self.factory.post('/', **self.extra) + request = self.factory.post("/", **self.extra) request.session = self.client.session response = view(request) self.assertEqual(response.status_code, 201) - self.assertNotEqual(response.data['odk_token'], old_token) + self.assertNotEqual(response.data["odk_token"], old_token) # Test that the previous token was set to inactive - inactive_token = ODKToken.objects.get( - user=self.user, status=ODKToken.INACTIVE) + inactive_token = ODKToken.objects.get(user=self.user, status=ODKToken.INACTIVE) self.assertEqual(inactive_token.raw_key, old_token) def test_retrieve_odk_token(self): """ Test that ODK Tokens can be retrieved """ - view = ConnectViewSet.as_view({ - 'post': 'odk_token', - 'get': 'odk_token' - }) - request = self.factory.post('/', **self.extra) + view = ConnectViewSet.as_view({"post": "odk_token", "get": "odk_token"}) + request = self.factory.post("/", **self.extra) request.session = self.client.session response = view(request) self.assertEqual(response.status_code, 201) - odk_token = response.data['odk_token'] - expires = response.data['expires'] + odk_token = response.data["odk_token"] + expires = response.data["expires"] - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.session = self.client.session response = view(request) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['odk_token'], odk_token) - self.assertEqual(response.data['expires'], expires) + self.assertEqual(response.data["odk_token"], odk_token) + self.assertEqual(response.data["expires"], expires) def test_deactivates_multiple_active_odk_token(self): """ Test that the viewset deactivates tokens when two or more are active at the same time and returns a new token """ - view = ConnectViewSet.as_view({ - 'post': 'odk_token', - 'get': 'odk_token' - }) + view = ConnectViewSet.as_view({"post": "odk_token", "get": "odk_token"}) # Create two active tokens token_1 = ODKToken.objects.create(user=self.user) @@ -590,18 +593,17 @@ def test_deactivates_multiple_active_odk_token(self): # Test that the GET request deactivates the two active tokens # and returns a new active token - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.session = self.client.session response = view(request) self.assertEqual(response.status_code, 200) - self.assertEqual( - len(ODKToken.objects.filter(status=ODKToken.ACTIVE)), 1) - self.assertNotEqual(response.data['odk_token'], token_1.raw_key) - self.assertNotEqual(response.data['odk_token'], token_2.raw_key) + self.assertEqual(len(ODKToken.objects.filter(status=ODKToken.ACTIVE)), 1) + self.assertNotEqual(response.data["odk_token"], token_1.raw_key) + self.assertNotEqual(response.data["odk_token"], token_2.raw_key) token_1 = ODKToken.objects.get(pk=token_1.pk) token_2 = ODKToken.objects.get(pk=token_2.pk) - token_3_key = response.data['odk_token'] + token_3_key = response.data["odk_token"] self.assertEqual(token_1.status, ODKToken.INACTIVE) self.assertEqual(token_2.status, ODKToken.INACTIVE) @@ -612,13 +614,11 @@ def test_deactivates_multiple_active_odk_token(self): token_1.status = ODKToken.ACTIVE token_1.save() - self.assertEqual( - len(ODKToken.objects.filter(status=ODKToken.ACTIVE)), 2) - request = self.factory.post('/', **self.extra) + self.assertEqual(len(ODKToken.objects.filter(status=ODKToken.ACTIVE)), 2) + request = self.factory.post("/", **self.extra) request.session = self.client.session response = view(request) self.assertEqual(response.status_code, 201) - self.assertEqual( - len(ODKToken.objects.filter(status=ODKToken.ACTIVE)), 1) - self.assertNotEqual(response.data['odk_token'], token_1.raw_key) - self.assertNotEqual(response.data['odk_token'], token_3_key) + self.assertEqual(len(ODKToken.objects.filter(status=ODKToken.ACTIVE)), 1) + self.assertNotEqual(response.data["odk_token"], token_1.raw_key) + self.assertNotEqual(response.data["odk_token"], token_3_key) diff --git a/onadata/apps/api/tests/viewsets/test_data_viewset.py b/onadata/apps/api/tests/viewsets/test_data_viewset.py index 0780e6fe40..ff2140f9a7 100644 --- a/onadata/apps/api/tests/viewsets/test_data_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_data_viewset.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Test /data API endpoint implementation. +""" from __future__ import unicode_literals import datetime @@ -21,15 +25,18 @@ from httmock import urlmatch, HTTMock from mock import patch -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - enketo_urls_mock +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import enketo_urls_mock from onadata.apps.api.viewsets.data_viewset import DataViewSet from onadata.apps.api.viewsets.project_viewset import ProjectViewSet from onadata.apps.api.viewsets.xform_viewset import XFormViewSet -from onadata.apps.logger.models import \ - Instance, SurveyType, XForm, Attachment, SubmissionReview +from onadata.apps.logger.models import ( + Instance, + SurveyType, + XForm, + Attachment, + SubmissionReview, +) from onadata.apps.logger.models.instance import InstanceHistory from onadata.apps.logger.models.instance import get_attachment_url from onadata.apps.main import tests as main_tests @@ -38,74 +45,85 @@ from onadata.apps.main.tests.test_base import TestBase from onadata.apps.messaging.constants import XFORM, SUBMISSION_DELETED from onadata.libs import permissions as role -from onadata.libs.permissions import ReadOnlyRole, EditorRole, \ - EditorMinorRole, DataEntryOnlyRole, DataEntryMinorRole, \ - ManagerRole -from onadata.libs.serializers.submission_review_serializer import \ - SubmissionReviewSerializer +from onadata.libs.permissions import ( + ReadOnlyRole, + EditorRole, + EditorMinorRole, + DataEntryOnlyRole, + DataEntryMinorRole, + ManagerRole, +) +from onadata.libs.serializers.submission_review_serializer import ( + SubmissionReviewSerializer, +) from onadata.libs.utils.common_tags import MONGO_STRFTIME from onadata.libs.utils.logger_tools import create_instance -@urlmatch(netloc=r'(.*\.)?enketo\.ona\.io$') +@urlmatch(netloc=r"(.*\.)?enketo\.ona\.io$") def enketo_edit_mock(url, request): response = requests.Response() response.status_code = 201 response._content = ( '{"edit_url": "https://hmh2a.enketo.ona.io/edit/XA0bG8D' - 'f?instance_id=672927e3-9ad4-42bb-9538-388ea1fb6699&retu' - 'rnUrl=http://test.io/test_url", "code": 201}') + "f?instance_id=672927e3-9ad4-42bb-9538-388ea1fb6699&retu" + 'rnUrl=http://test.io/test_url", "code": 201}' + ) return response -@urlmatch(netloc=r'(.*\.)?enketo\.ona\.io$') +@urlmatch(netloc=r"(.*\.)?enketo\.ona\.io$") def enketo_mock_http_413(url, request): response = requests.Response() response.status_code = 413 - response._content = '' + response._content = "" return response def _data_list(formid): - return [{ - 'id': formid, - 'id_string': 'transportation_2011_07_25', - 'title': 'transportation_2011_07_25', - 'description': '', - 'url': 'http://testserver/api/v1/data/%s' % formid - }] + return [ + { + "id": formid, + "id_string": "transportation_2011_07_25", + "title": "transportation_2011_07_25", + "description": "", + "url": "http://testserver/api/v1/data/%s" % formid, + } + ] def _data_instance(dataid): return { - '_bamboo_dataset_id': '', - '_attachments': [], - '_geolocation': [None, None], - '_xform_id_string': 'transportation_2011_07_25', - 'transport/available_transportation_types_to_referral_facility': - 'none', - '_status': 'submitted_via_web', - '_id': dataid + "_bamboo_dataset_id": "", + "_attachments": [], + "_geolocation": [None, None], + "_xform_id_string": "transportation_2011_07_25", + "transport/available_transportation_types_to_referral_facility": "none", + "_status": "submitted_via_web", + "_id": dataid, } +# pylint: disable=too-many-public-methods class TestDataViewSet(TestBase): + """ + Test /data API endpoint implementation. + """ def setUp(self): super(self.__class__, self).setUp() self._create_user_and_login() self._publish_transportation_form() self.factory = RequestFactory() - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} def test_data(self): """Test DataViewSet list""" self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) formid = self.xform.pk data = _data_list(formid) @@ -115,31 +133,31 @@ def test_data(self): self.assertIsInstance(response.data, list) self.assertTrue(self.xform.instances.count()) - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk data = _data_instance(dataid) self.assertDictContainsSubset( - data, sorted(response.data, key=lambda x: x['_id'])[0]) + data, sorted(response.data, key=lambda x: x["_id"])[0] + ) data = { - '_xform_id_string': 'transportation_2011_07_25', - 'transport/available_transportation_types_to_referral_facility': - 'none', - '_submitted_by': 'bob', + "_xform_id_string": "transportation_2011_07_25", + "transport/available_transportation_types_to_referral_facility": "none", + "_submitted_by": "bob", } - view = DataViewSet.as_view({'get': 'retrieve'}) + view = DataViewSet.as_view({"get": "retrieve"}) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertIsInstance(response.data, dict) self.assertDictContainsSubset(data, response.data) @override_settings(STREAM_DATA=True) def test_data_streaming(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) formid = self.xform.pk data = _data_list(formid) @@ -149,122 +167,124 @@ def test_data_streaming(self): response = view(request, pk=formid) self.assertEqual(response.status_code, 200) streaming_data = json.loads( - ''.join([i.decode('utf-8') for i in response.streaming_content]) + "".join([i.decode("utf-8") for i in response.streaming_content]) ) self.assertIsInstance(streaming_data, list) self.assertTrue(self.xform.instances.count()) - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk data = _data_instance(dataid) self.assertDictContainsSubset( - data, sorted(streaming_data, key=lambda x: x['_id'])[0]) + data, sorted(streaming_data, key=lambda x: x["_id"])[0] + ) data = { - '_xform_id_string': 'transportation_2011_07_25', - 'transport/available_transportation_types_to_referral_facility': - 'none', - '_submitted_by': 'bob', + "_xform_id_string": "transportation_2011_07_25", + "transport/available_transportation_types_to_referral_facility": "none", + "_submitted_by": "bob", } - view = DataViewSet.as_view({'get': 'retrieve'}) + view = DataViewSet.as_view({"get": "retrieve"}) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertIsInstance(response.data, dict) self.assertDictContainsSubset(data, response.data) def test_catch_data_error(self): - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk - query_str = [('\'{"_submission_time":{' - '"$and":[{"$gte":"2015-11-15T00:00:00"},' - '{"$lt":"2015-11-16T00:00:00"}]}}')] + query_str = [ + ( + '\'{"_submission_time":{' + '"$and":[{"$gte":"2015-11-15T00:00:00"},' + '{"$lt":"2015-11-16T00:00:00"}]}}' + ) + ] data = { - 'query': query_str, + "query": query_str, } - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data.get('detail'), - 'invalid regular expression: invalid character range\n') + response.data.get("detail"), + "invalid regular expression: invalid character range\n", + ) def test_data_list_with_xform_in_delete_async_queue(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) initial_count = len(response.data) self.xform.deleted_at = timezone.now() self.xform.save() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(len(response.data), initial_count - 1) def test_numeric_types_are_rendered_as_required(self): tutorial_folder = os.path.join( - os.path.dirname(__file__), - '..', 'fixtures', 'forms', 'tutorial') - self._publish_xls_file_and_set_xform(os.path.join(tutorial_folder, - 'tutorial.xlsx')) + os.path.dirname(__file__), "..", "fixtures", "forms", "tutorial" + ) + self._publish_xls_file_and_set_xform( + os.path.join(tutorial_folder, "tutorial.xlsx") + ) - instance_path = os.path.join(tutorial_folder, 'instances', '1.xml') - create_instance(self.user.username, open(instance_path, 'rb'), []) + instance_path = os.path.join(tutorial_folder, "instances", "1.xml") + create_instance(self.user.username, open(instance_path, "rb"), []) self.assertEqual(self.xform.instances.count(), 1) - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.xform.id) self.assertEqual(response.status_code, 200) # check that ONLY values with numeric and decimal types are converted - self.assertEqual(response.data[0].get('age'), 35) - self.assertEqual(response.data[0].get('net_worth'), 100000.00) - self.assertEqual(response.data[0].get('imei'), '351746052009472') + self.assertEqual(response.data[0].get("age"), 35) + self.assertEqual(response.data[0].get("net_worth"), 100000.00) + self.assertEqual(response.data[0].get("imei"), "351746052009472") def test_data_jsonp(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk - request = self.factory.get('/', **self.extra) - response = view(request, pk=formid, format='jsonp') + request = self.factory.get("/", **self.extra) + response = view(request, pk=formid, format="jsonp") self.assertEqual(response.status_code, 200) response.render() - content = response.content.decode('utf-8') - self.assertTrue(content.startswith('callback(')) - self.assertTrue(content.endswith(');')) + content = response.content.decode("utf-8") + self.assertTrue(content.startswith("callback(")) + self.assertTrue(content.endswith(");")) self.assertEqual(len(response.data), 4) def _assign_user_role(self, user, role): # share bob's project with alice and give alice an editor role - data = {'username': user.username, 'role': role.name} - request = self.factory.put('/', data=data, **self.extra) - xform_view = XFormViewSet.as_view({ - 'put': 'share' - }) + data = {"username": user.username, "role": role.name} + request = self.factory.put("/", data=data, **self.extra) + xform_view = XFormViewSet.as_view({"put": "share"}) response = xform_view(request, pk=self.xform.pk) self.assertEqual(response.status_code, 204) - self.assertTrue( - role.user_has_role(user, self.xform) - ) + self.assertTrue(role.user_has_role(user, self.xform)) def test_returned_data_is_based_on_form_permissions(self): # create a form and make submissions to it self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) # create user alice - user_alice = self._create_user('alice', 'alice') + user_alice = self._create_user("alice", "alice") # create user profile and set require_auth to false for tests profile, created = UserProfile.objects.get_or_create(user=user_alice) profile.require_auth = False @@ -276,23 +296,21 @@ def test_returned_data_is_based_on_form_permissions(self): self._assign_user_role(user_alice, DataEntryOnlyRole) - alices_extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % user_alice.auth_token.key - } + alices_extra = {"HTTP_AUTHORIZATION": "Token %s" % user_alice.auth_token.key} - request = self.factory.get('/', **alices_extra) + request = self.factory.get("/", **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) data = {"start": 1, "limit": 4} - request = self.factory.get('/', data=data, **alices_extra) + request = self.factory.get("/", data=data, **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) data = {"sort": 1} - request = self.factory.get('/', data=data, **alices_extra) + request = self.factory.get("/", data=data, **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) @@ -300,7 +318,7 @@ def test_returned_data_is_based_on_form_permissions(self): self._assign_user_role(user_alice, EditorMinorRole) # check that by default, alice can be able to access all the data - request = self.factory.get('/', **alices_extra) + request = self.factory.get("/", **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) @@ -317,33 +335,39 @@ def test_returned_data_is_based_on_form_permissions(self): # check that 2 instances were 'submitted by' alice instances_submitted_by_alice = self.xform.instances.filter( - user=user_alice).count() + user=user_alice + ).count() self.assertTrue(instances_submitted_by_alice, 2) # check that alice will only be able to see the data she submitted - request = self.factory.get('/', **alices_extra) + request = self.factory.get("/", **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) # check that alice views only her data when querying data - data = {"start": 0, "limit": 100, "sort": '{"_submission_time":1}', - "query": "c"} - request = self.factory.get('/', data=data, **alices_extra) + data = { + "start": 0, + "limit": 100, + "sort": '{"_submission_time":1}', + "query": "c", + } + request = self.factory.get("/", data=data, **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) self.assertListEqual( - [r['_submitted_by'] for r in response.data], ['alice', 'alice']) + [r["_submitted_by"] for r in response.data], ["alice", "alice"] + ) data = {"start": 1, "limit": 1} - request = self.factory.get('/', data=data, **alices_extra) + request = self.factory.get("/", data=data, **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) data = {"sort": 1} - request = self.factory.get('/', data=data, **alices_extra) + request = self.factory.get("/", data=data, **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) @@ -354,14 +378,14 @@ def test_returned_data_is_based_on_form_permissions(self): self._assign_user_role(user_alice, EditorRole) - request = self.factory.get('/', **alices_extra) + request = self.factory.get("/", **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) self._assign_user_role(user_alice, ReadOnlyRole) - request = self.factory.get('/', **alices_extra) + request = self.factory.get("/", **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) @@ -369,55 +393,44 @@ def test_returned_data_is_based_on_form_permissions(self): def test_xform_meta_permissions_not_affected_w_projects_perms(self): # create a form and make submissions to it self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) # create user alice - user_alice = self._create_user('alice', 'alice') + user_alice = self._create_user("alice", "alice") # create user profile and set require_auth to false for tests profile, created = UserProfile.objects.get_or_create(user=user_alice) profile.require_auth = False profile.save() - data = {'username': user_alice.username, 'role': EditorRole.name} - request = self.factory.put('/', data=data, **self.extra) - project_view = ProjectViewSet.as_view({ - 'put': 'share' - }) + data = {"username": user_alice.username, "role": EditorRole.name} + request = self.factory.put("/", data=data, **self.extra) + project_view = ProjectViewSet.as_view({"put": "share"}) response = project_view(request, pk=self.project.pk) self.assertEqual(response.status_code, 204) - self.assertTrue( - EditorRole.user_has_role(user_alice, self.xform) - ) + self.assertTrue(EditorRole.user_has_role(user_alice, self.xform)) self._assign_user_role(user_alice, EditorMinorRole) - MetaData.xform_meta_permission(self.xform, - data_value='editor-minor|dataentry') + MetaData.xform_meta_permission(self.xform, data_value="editor-minor|dataentry") - self.assertFalse( - EditorRole.user_has_role(user_alice, self.xform) - ) + self.assertFalse(EditorRole.user_has_role(user_alice, self.xform)) - alices_extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % user_alice.auth_token.key - } + alices_extra = {"HTTP_AUTHORIZATION": "Token %s" % user_alice.auth_token.key} - request = self.factory.get('/', **alices_extra) + request = self.factory.get("/", **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - self.assertFalse( - EditorRole.user_has_role(user_alice, self.xform) - ) + self.assertFalse(EditorRole.user_has_role(user_alice, self.xform)) self.assertEqual(len(response.data), 0) def test_data_entryonly_can_submit_but_not_view(self): # create user alice - user_alice = self._create_user('alice', 'alice') + user_alice = self._create_user("alice", "alice") # create user profile and set require_auth to false for tests profile, created = UserProfile.objects.get_or_create(user=user_alice) profile.require_auth = False @@ -430,229 +443,233 @@ def test_data_entryonly_can_submit_but_not_view(self): DataEntryOnlyRole.add(user_alice, self.xform) DataEntryOnlyRole.add(user_alice, self.project) - auth = DigestAuth('alice', 'alice') + auth = DigestAuth("alice", "alice") - paths = [os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'instances', s, s + '.xml') for s in self.surveys] + paths = [ + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + for s in self.surveys + ] for path in paths: self._make_submission(path, auth=auth) - alices_extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % user_alice.auth_token.key - } + alices_extra = {"HTTP_AUTHORIZATION": "Token %s" % user_alice.auth_token.key} - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk - request = self.factory.get('/', **alices_extra) + request = self.factory.get("/", **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 404) DataEntryMinorRole.add(user_alice, self.xform) DataEntryMinorRole.add(user_alice, self.project) - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk - request = self.factory.get('/', **alices_extra) + request = self.factory.get("/", **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) def test_data_pagination(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk # no page param no pagination - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) - request = self.factory.get('/', data={"page": "1", "page_size": 2}, - **self.extra) + request = self.factory.get( + "/", data={"page": "1", "page_size": 2}, **self.extra + ) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) # Link pagination headers are present and work as intended - self.assertIn('Link', response) + self.assertIn("Link", response) self.assertEqual( - response['Link'], - '; rel="next"' + response["Link"], '; rel="next"' ) - request = self.factory.get( - '/', data={"page": 2, "page_size": 1}, **self.extra) + request = self.factory.get("/", data={"page": 2, "page_size": 1}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - self.assertIn('Link', response) + self.assertIn("Link", response) self.assertEqual( - response['Link'], - ('; rel="prev", ' - '; rel="next", ' - '; rel="last", ' - '; rel="first"')) + response["Link"], + ( + '; rel="prev", ' + '; rel="next", ' + '; rel="last", ' + '; rel="first"' + ), + ) - request = self.factory.get('/', data={"page_size": "3"}, **self.extra) + request = self.factory.get("/", data={"page_size": "3"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 3) request = self.factory.get( - '/', data={"page": "1", "page_size": "2"}, **self.extra) + "/", data={"page": "1", "page_size": "2"}, **self.extra + ) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) # invalid page returns a 404 - request = self.factory.get('/', data={"page": "invalid"}, **self.extra) + request = self.factory.get("/", data={"page": "invalid"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 404) # invalid page size is ignored - request = self.factory.get('/', data={"page_size": "invalid"}, - **self.extra) + request = self.factory.get("/", data={"page_size": "invalid"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) # Query param returns correct pagination headers request = self.factory.get( - '/', data={"page_size": "1", "query": "ambulance"}, - **self.extra) + "/", data={"page_size": "1", "query": "ambulance"}, **self.extra + ) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - self.assertIn('Link', response) + self.assertIn("Link", response) self.assertEqual( - response['Link'], - ('; rel="next"')) + response["Link"], ('; rel="next"') + ) def test_sort_query_param_with_invalid_values(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk # without sort param - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) - error_message = ('Expecting property name enclosed in ' - 'double quotes: line 1 column 2 (char 1)') + error_message = ( + "Expecting property name enclosed in " + "double quotes: line 1 column 2 (char 1)" + ) - request = self.factory.get('/', data={"sort": '{'':}'}, - **self.extra) + request = self.factory.get("/", data={"sort": "{" ":}"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data.get('detail'), error_message) + self.assertEqual(response.data.get("detail"), error_message) - request = self.factory.get('/', data={"sort": '{:}'}, - **self.extra) + request = self.factory.get("/", data={"sort": "{:}"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data.get('detail'), error_message) + self.assertEqual(response.data.get("detail"), error_message) - request = self.factory.get('/', data={"sort": '{'':''}'}, - **self.extra) + request = self.factory.get("/", data={"sort": "{" ":" "}"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data.get('detail'), error_message) + self.assertEqual(response.data.get("detail"), error_message) # test sort with a key that os likely in the json data - request = self.factory.get('/', data={"sort": 'random'}, - **self.extra) + request = self.factory.get("/", data={"sort": "random"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) def test_data_start_limit_no_records(self): - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk # no start, limit params - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) - request = self.factory.get('/', data={"start": "1", "limit": 2}, - **self.extra) + request = self.factory.get("/", data={"start": "1", "limit": 2}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) def test_data_start_limit(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk # no start, limit params - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) - self.assertTrue(response.has_header('ETag')) - etag_data = response['Etag'] + self.assertTrue(response.has_header("ETag")) + etag_data = response["Etag"] - request = self.factory.get('/', data={"start": "1", "limit": 2}, - **self.extra) + request = self.factory.get("/", data={"start": "1", "limit": 2}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) - self.assertNotEqual(etag_data, response['Etag']) - etag_data = response['Etag'] + self.assertNotEqual(etag_data, response["Etag"]) + etag_data = response["Etag"] response.render() data = json.loads(response.content) - self.assertEqual([i['_uuid'] for i in data], - ['f3d8dc65-91a6-4d0f-9e97-802128083390', - '9c6f3468-cfda-46e8-84c1-75458e72805d']) + self.assertEqual( + [i["_uuid"] for i in data], + [ + "f3d8dc65-91a6-4d0f-9e97-802128083390", + "9c6f3468-cfda-46e8-84c1-75458e72805d", + ], + ) - request = self.factory.get('/', data={"start": "3", "limit": 1}, - **self.extra) + request = self.factory.get("/", data={"start": "3", "limit": 1}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) - self.assertNotEqual(etag_data, response['Etag']) - etag_data = response['Etag'] + self.assertNotEqual(etag_data, response["Etag"]) + etag_data = response["Etag"] response.render() data = json.loads(response.content) - self.assertEqual([i['_uuid'] for i in data], - ['9f0a1508-c3b7-4c99-be00-9b237c26bcbf']) + self.assertEqual( + [i["_uuid"] for i in data], ["9f0a1508-c3b7-4c99-be00-9b237c26bcbf"] + ) - request = self.factory.get('/', data={"limit": "3"}, **self.extra) + request = self.factory.get("/", data={"limit": "3"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 3) - self.assertNotEqual(etag_data, response['Etag']) - etag_data = response['Etag'] + self.assertNotEqual(etag_data, response["Etag"]) + etag_data = response["Etag"] - request = self.factory.get( - '/', data={"start": "1", "limit": "2"}, **self.extra) + request = self.factory.get("/", data={"start": "1", "limit": "2"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) - self.assertNotEqual(etag_data, response['Etag']) + self.assertNotEqual(etag_data, response["Etag"]) # invalid start is ignored, all data is returned - request = self.factory.get('/', data={"start": "invalid"}, - **self.extra) + request = self.factory.get("/", data={"start": "invalid"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) # invalid limit is ignored, all data is returned - request = self.factory.get('/', data={"limit": "invalid"}, - **self.extra) + request = self.factory.get("/", data={"limit": "invalid"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) # invalid start is ignored, all data is returned - request = self.factory.get('/', data={"start": "", "limit": 10}, - **self.extra) + request = self.factory.get("/", data={"start": "", "limit": 10}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) @@ -664,97 +681,93 @@ def test_paginate_and_sort_streaming_data(self): responses """ self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk # will result in a queryset due to the page and page_size params # hence paging and thus len(self.object_list) for length - query_data = { - "page_size": 3, - "page": 1, - "sort": '{"date_created":-1}' - } - request = self.factory.get('/', data=query_data, - **self.extra) + query_data = {"page_size": 3, "page": 1, "sort": '{"date_created":-1}'} + request = self.factory.get("/", data=query_data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - streaming_data = json.loads(''.join( - [c.decode('utf-8') for c in response.streaming_content])) + streaming_data = json.loads( + "".join([c.decode("utf-8") for c in response.streaming_content]) + ) self.assertEqual(len(streaming_data), 3) # Test `date_created` field is sorted correctly - expected_order = list(Instance.objects.filter( - xform=self.xform).order_by( - '-date_created').values_list('id', flat=True)) - items_in_order = [sub.get('_id') for sub in streaming_data] + expected_order = list( + Instance.objects.filter(xform=self.xform) + .order_by("-date_created") + .values_list("id", flat=True) + ) + items_in_order = [sub.get("_id") for sub in streaming_data] self.assertEqual(expected_order[:3], items_in_order) - self.assertTrue(response.has_header('ETag')) + self.assertTrue(response.has_header("ETag")) # Data fetched for page 2 should return a different list - data = { - "page_size": 3, - "page": 2, - "sort": '{"date_created":-1}' - } - request2 = self.factory.get('/', data=data, - **self.extra) + data = {"page_size": 3, "page": 2, "sort": '{"date_created":-1}'} + request2 = self.factory.get("/", data=data, **self.extra) response2 = view(request2, pk=formid) self.assertEqual(response2.status_code, 200) - streaming_data = json.loads(''.join( - [c.decode('utf-8') for c in response2.streaming_content])) + streaming_data = json.loads( + "".join([c.decode("utf-8") for c in response2.streaming_content]) + ) self.assertEqual(len(streaming_data), 1) - pg_2_items_in_order = [sub.get('_id') for sub in streaming_data] + pg_2_items_in_order = [sub.get("_id") for sub in streaming_data] self.assertEqual(expected_order[3:], pg_2_items_in_order) def test_data_start_limit_sort(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk data = {"start": 1, "limit": 2, "sort": '{"_id":1}'} - request = self.factory.get('/', data=data, - **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) - self.assertTrue(response.has_header('ETag')) + self.assertTrue(response.has_header("ETag")) response.render() data = json.loads(response.content) - self.assertEqual([i['_uuid'] for i in data], - ['f3d8dc65-91a6-4d0f-9e97-802128083390', - '9c6f3468-cfda-46e8-84c1-75458e72805d']) + self.assertEqual( + [i["_uuid"] for i in data], + [ + "f3d8dc65-91a6-4d0f-9e97-802128083390", + "9c6f3468-cfda-46e8-84c1-75458e72805d", + ], + ) def test_filter_pending_submission_reviews(self): """ Test that a user is able to query for null """ self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) # Create approved submission review for one instance - instances = self.xform.instances.all().order_by('pk') + instances = self.xform.instances.all().order_by("pk") instance = instances[0] self.assertFalse(instance.has_a_review) - self._create_user_and_login('bob', '1234') - ManagerRole.add( - self.user, - self.xform.instances.all().order_by('pk')[0].xform) + self._create_user_and_login("bob", "1234") + ManagerRole.add(self.user, self.xform.instances.all().order_by("pk")[0].xform) data = { - 'note': "Approved!", + "note": "Approved!", "instance": instance.id, - "status": SubmissionReview.APPROVED - } + "status": SubmissionReview.APPROVED, + } - serializer_instance = SubmissionReviewSerializer(data=data, context={ - "request": request}) + serializer_instance = SubmissionReviewSerializer( + data=data, context={"request": request} + ) serializer_instance.is_valid() serializer_instance.save() instance.refresh_from_db() @@ -762,15 +775,13 @@ def test_filter_pending_submission_reviews(self): # Confirm xform submission review enabled self.assertTrue(instance.has_a_review) # Confirm instance json now has _review_status field - self.assertIn('_review_status', dict(instance.json)) + self.assertIn("_review_status", dict(instance.json)) # Confirm instance submission review status - self.assertEquals('1', instance.json['_review_status']) + self.assertEqual("1", instance.json["_review_status"]) # Query xform data by submission review status 3 query_str = '{"_review_status": 3}' - request = self.factory.get( - f'/?query={query_str}', - **self.extra) + request = self.factory.get(f"/?query={query_str}", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) @@ -779,9 +790,7 @@ def test_filter_pending_submission_reviews(self): # Query xform data by NULL submission review query_str = '{"_review_status": null}' - request = self.factory.get( - f'/?query={query_str}', - **self.extra) + request = self.factory.get(f"/?query={query_str}", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 3) @@ -789,26 +798,30 @@ def test_filter_pending_submission_reviews(self): @override_settings(STREAM_DATA=True) def test_data_start_limit_sort_json_field(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk # will result in a generator due to the JSON sort # hence self.total_count will be used for length in streaming response data = { "start": 1, "limit": 2, - "sort": '{"transport/available_transportation_types_to_referral_facility":1}' # noqa + "sort": '{"transport/available_transportation_types_to_referral_facility":1}', # noqa } - request = self.factory.get('/', data=data, - **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - self.assertTrue(response.has_header('ETag')) - data = json.loads(''.join([ - c.decode('utf-8') for c in response.streaming_content])) + self.assertTrue(response.has_header("ETag")) + data = json.loads( + "".join([c.decode("utf-8") for c in response.streaming_content]) + ) self.assertEqual(len(data), 2) - self.assertEqual([i['_uuid'] for i in data], - ['f3d8dc65-91a6-4d0f-9e97-802128083390', - '5b2cc313-fc09-437e-8149-fcd32f695d41']) + self.assertEqual( + [i["_uuid"] for i in data], + [ + "f3d8dc65-91a6-4d0f-9e97-802128083390", + "5b2cc313-fc09-437e-8149-fcd32f695d41", + ], + ) # will result in a queryset due to the page and page_size params # hence paging and thus len(self.object_list) for length @@ -816,30 +829,30 @@ def test_data_start_limit_sort_json_field(self): "page": 1, "page_size": 2, } - request = self.factory.get('/', data=data, - **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - data = json.loads(''.join( - [c.decode('utf-8') for c in response.streaming_content])) + data = json.loads( + "".join([c.decode("utf-8") for c in response.streaming_content]) + ) self.assertEqual(len(data), 2) data = { "page": 1, "page_size": 3, } - request = self.factory.get('/', data=data, - **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - data = json.loads(''.join( - [c.decode('utf-8') for c in response.streaming_content])) + data = json.loads( + "".join([c.decode("utf-8") for c in response.streaming_content]) + ) self.assertEqual(len(data), 3) def test_data_anon(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/') + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/") formid = self.xform.pk response = view(request, pk=formid) # data not found for anonymous access to private data @@ -858,19 +871,19 @@ def test_data_anon(self): self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) self.assertTrue(self.xform.instances.count()) - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk data = _data_instance(dataid) self.assertDictContainsSubset( - data, sorted(response.data, key=lambda x: x['_id'])[0]) + data, sorted(response.data, key=lambda x: x["_id"])[0] + ) data = { - '_xform_id_string': 'transportation_2011_07_25', - 'transport/available_transportation_types_to_referral_facility': - 'none', - '_submitted_by': 'bob', + "_xform_id_string": "transportation_2011_07_25", + "transport/available_transportation_types_to_referral_facility": "none", + "_submitted_by": "bob", } - view = DataViewSet.as_view({'get': 'retrieve'}) + view = DataViewSet.as_view({"get": "retrieve"}) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, dict) @@ -878,53 +891,53 @@ def test_data_anon(self): def test_data_public(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) - response = view(request, pk='public') + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) + response = view(request, pk="public") self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) self.xform.shared_data = True self.xform.save() formid = self.xform.pk data = _data_list(formid) - response = view(request, pk='public') + response = view(request, pk="public") self.assertEqual(response.status_code, 200) self.assertEqual(response.data, data) def test_data_public_anon_user(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/') - response = view(request, pk='public') + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/") + response = view(request, pk="public") self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) self.xform.shared_data = True self.xform.save() formid = self.xform.pk data = _data_list(formid) - response = view(request, pk='public') + response = view(request, pk="public") self.assertEqual(response.status_code, 200) self.assertEqual(response.data, data) def test_data_user_public(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) - response = view(request, pk='public') + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) + response = view(request, pk="public") self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) self.xform.shared_data = True self.xform.save() formid = self.xform.pk data = _data_list(formid) - response = view(request, pk='public') + response = view(request, pk="public") self.assertEqual(response.status_code, 200) self.assertEqual(response.data, data) def test_data_bad_formid(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) formid = self.xform.pk @@ -941,14 +954,14 @@ def test_data_bad_formid(self): formid = "INVALID" response = view(request, pk=formid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) error_message = "Invalid form ID. It must be a positive integer" - self.assertEqual(str(response.data['detail']), error_message) + self.assertEqual(str(response.data["detail"]), error_message) def test_data_bad_dataid(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) formid = self.xform.pk @@ -958,119 +971,116 @@ def test_data_bad_dataid(self): self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) self.assertTrue(self.xform.instances.count()) - dataid = 'INVALID' + dataid = "INVALID" data = _data_instance(dataid) - view = DataViewSet.as_view({'get': 'retrieve'}) + view = DataViewSet.as_view({"get": "retrieve"}) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) def test_filter_by_submission_time_and_submitted_by(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk - instance = self.xform.instances.all().order_by('pk')[0] + instance = self.xform.instances.all().order_by("pk")[0] response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) submission_time = instance.date_created.strftime(MONGO_STRFTIME) - query_str = ('{"_submission_time": {"$gte": "%s"},' - ' "_submitted_by": "%s"}' % (submission_time, 'bob')) - data = { - 'query': query_str, - 'limit': 2, - 'sort': [] - } - request = self.factory.get('/', data=data, **self.extra) + query_str = '{"_submission_time": {"$gte": "%s"},' ' "_submitted_by": "%s"}' % ( + submission_time, + "bob", + ) + data = {"query": query_str, "limit": 2, "sort": []} + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) def test_filter_by_date_modified(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk - instance = self.xform.instances.all().order_by('pk')[0] + instance = self.xform.instances.all().order_by("pk")[0] response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) - instance = self.xform.instances.all().order_by('-date_created')[0] + instance = self.xform.instances.all().order_by("-date_created")[0] date_modified = instance.date_modified.isoformat() - query_str = ('{"_date_modified": {"$gte": "%s"},' - ' "_submitted_by": "%s"}' % (date_modified, 'bob')) - data = { - 'query': query_str - } - request = self.factory.get('/', data=data, **self.extra) + query_str = '{"_date_modified": {"$gte": "%s"},' ' "_submitted_by": "%s"}' % ( + date_modified, + "bob", + ) + data = {"query": query_str} + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) expected_count = self.xform.instances.filter( - date_modified__gte=date_modified).count() + date_modified__gte=date_modified + ).count() self.assertEqual(len(response.data), expected_count) def test_filter_by_submission_time_and_submitted_by_with_data_arg(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk - instance = self.xform.instances.all().order_by('pk')[0] + instance = self.xform.instances.all().order_by("pk")[0] response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) submission_time = instance.date_created.strftime(MONGO_STRFTIME) - query_str = ('{"_submission_time": {"$gte": "%s"},' - ' "_submitted_by": "%s"}' % (submission_time, 'bob')) - data = { - 'data': query_str, - 'limit': 2, - 'sort': [] - } - request = self.factory.get('/', data=data, **self.extra) + query_str = '{"_submission_time": {"$gte": "%s"},' ' "_submitted_by": "%s"}' % ( + submission_time, + "bob", + ) + data = {"data": query_str, "limit": 2, "sort": []} + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) def test_filter_by_submission_time_date_formats(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk - data = {'query': '{"_submission_time":{"$gt":"2018-04-19"}}'} - request = self.factory.get('/', data=data, **self.extra) + data = {"query": '{"_submission_time":{"$gt":"2018-04-19"}}'} + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - data = {'query': '{"_submission_time":{"$gt":"2018-04-19T14:46:32"}}'} - request = self.factory.get('/', data=data, **self.extra) + data = {"query": '{"_submission_time":{"$gt":"2018-04-19T14:46:32"}}'} + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) def test_data_with_query_parameter(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk - instance = self.xform.instances.all().order_by('pk')[0] + instance = self.xform.instances.all().order_by("pk")[0] dataid = instance.pk response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) query_str = '{"_id": "%s"}' % dataid - request = self.factory.get('/?query=%s' % query_str, **self.extra) + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) submission_time = instance.date_created.strftime(MONGO_STRFTIME) query_str = '{"_submission_time": {"$gte": "%s"}}' % submission_time - request = self.factory.get('/?query=%s' % query_str, **self.extra) + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) @@ -1088,47 +1098,55 @@ def test_data_with_query_parameter(self): first_datetime = start_time.strftime(MONGO_STRFTIME) second_datetime = start_time + timedelta(days=1, hours=20) - query_str = '{"_submission_time": {"$gte": "'\ - + first_datetime + '", "$lte": "'\ - + second_datetime.strftime(MONGO_STRFTIME) + '"}}' + query_str = ( + '{"_submission_time": {"$gte": "' + + first_datetime + + '", "$lte": "' + + second_datetime.strftime(MONGO_STRFTIME) + + '"}}' + ) - request = self.factory.get('/?query=%s' % query_str, **self.extra) + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) query_str = '{"_id: "%s"}' % dataid - request = self.factory.get('/?query=%s' % query_str, **self.extra) + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data.get('detail'), - u"Expecting ':' delimiter: line 1 column 9 (char 8)") + self.assertEqual( + response.data.get("detail"), + "Expecting ':' delimiter: line 1 column 9 (char 8)", + ) - query_str = '{"transport/available_transportation' \ - '_types_to_referral_facility": {"$i": "%s"}}' % "ambula" - request = self.factory.get('/?query=%s' % query_str, **self.extra) + query_str = ( + '{"transport/available_transportation' + '_types_to_referral_facility": {"$i": "%s"}}' % "ambula" + ) + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) # search a text - query_str = 'uuid:9f0a1508-c3b7-4c99-be00-9b237c26bcbf' - request = self.factory.get('/?query=%s' % query_str, **self.extra) + query_str = "uuid:9f0a1508-c3b7-4c99-be00-9b237c26bcbf" + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) # search an integer query_str = 7545 - request = self.factory.get('/?query=%s' % query_str, **self.extra) + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) def test_anon_data_list(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/') + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/") response = view(request) self.assertEqual(response.status_code, 200) @@ -1139,55 +1157,54 @@ def test_add_form_tag_propagates_to_data_tags(self): self._make_submissions() xform = XForm.objects.all()[0] pk = xform.id - view = XFormViewSet.as_view({ - 'get': 'labels', - 'post': 'labels', - 'delete': 'labels' - }) - data_view = DataViewSet.as_view({ - 'get': 'list', - }) + view = XFormViewSet.as_view( + {"get": "labels", "post": "labels", "delete": "labels"} + ) + data_view = DataViewSet.as_view( + { + "get": "list", + } + ) # no tags - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=pk) self.assertEqual(response.data, []) - request = self.factory.get('/', {'tags': 'hello'}, **self.extra) + request = self.factory.get("/", {"tags": "hello"}, **self.extra) response = data_view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) - request = self.factory.get('/', {'not_tagged': 'hello'}, **self.extra) + request = self.factory.get("/", {"not_tagged": "hello"}, **self.extra) response = data_view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) # add tag "hello" - request = self.factory.post('/', data={"tags": "hello"}, **self.extra) + request = self.factory.post("/", data={"tags": "hello"}, **self.extra) response = view(request, pk=pk) self.assertEqual(response.status_code, 201) - self.assertEqual(response.data, ['hello']) + self.assertEqual(response.data, ["hello"]) for i in self.xform.instances.all(): - self.assertIn('hello', i.tags.names()) + self.assertIn("hello", i.tags.names()) - request = self.factory.get('/', {'tags': 'hello'}, **self.extra) + request = self.factory.get("/", {"tags": "hello"}, **self.extra) response = data_view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) - request = self.factory.get('/', {'not_tagged': 'hello'}, **self.extra) + request = self.factory.get("/", {"not_tagged": "hello"}, **self.extra) response = data_view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) # remove tag "hello" - request = self.factory.delete('/', data={"tags": "hello"}, - **self.extra) - response = view(request, pk=pk, label='hello') + request = self.factory.delete("/", data={"tags": "hello"}, **self.extra) + response = view(request, pk=pk, label="hello") self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) for i in self.xform.instances.all(): - self.assertNotIn('hello', i.tags.names()) + self.assertNotIn("hello", i.tags.names()) def test_data_tags(self): """Test that when a tag is applied on an xform, @@ -1198,82 +1215,79 @@ def test_data_tags(self): pk = self.xform.pk i = self.xform.instances.all()[0] dataid = i.pk - data_view = DataViewSet.as_view({ - 'get': 'list', - }) - view = DataViewSet.as_view({ - 'get': 'labels', - 'post': 'labels', - 'delete': 'labels' - }) + data_view = DataViewSet.as_view( + { + "get": "list", + } + ) + view = DataViewSet.as_view( + {"get": "labels", "post": "labels", "delete": "labels"} + ) # no tags - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=pk, dataid=dataid) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) - request = self.factory.get('/', {'tags': 'hello'}, **self.extra) + request = self.factory.get("/", {"tags": "hello"}, **self.extra) response = data_view(request, pk=pk) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) - request = self.factory.get('/', {'not_tagged': 'hello'}, **self.extra) + request = self.factory.get("/", {"not_tagged": "hello"}, **self.extra) response = data_view(request, pk=pk) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), submission_count) # add tag "hello" - request = self.factory.post('/', data={"tags": "hello"}, **self.extra) + request = self.factory.post("/", data={"tags": "hello"}, **self.extra) response = view(request, pk=pk, dataid=dataid) self.assertEqual(response.status_code, 201) - self.assertEqual(response.data, ['hello']) - self.assertIn('hello', Instance.objects.get(pk=dataid).tags.names()) + self.assertEqual(response.data, ["hello"]) + self.assertIn("hello", Instance.objects.get(pk=dataid).tags.names()) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=pk, dataid=dataid) - self.assertEqual(response.data, ['hello']) + self.assertEqual(response.data, ["hello"]) - request = self.factory.get('/', {'tags': 'hello'}, **self.extra) + request = self.factory.get("/", {"tags": "hello"}, **self.extra) response = data_view(request, pk=pk) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) - request = self.factory.get('/', {'not_tagged': 'hello'}, **self.extra) + request = self.factory.get("/", {"not_tagged": "hello"}, **self.extra) response = data_view(request, pk=pk) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), submission_count - 1) # remove tag "hello" - request = self.factory.delete('/', **self.extra) - response = view(request, pk=pk, dataid=dataid, label='hello') + request = self.factory.delete("/", **self.extra) + response = view(request, pk=pk, dataid=dataid, label="hello") self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) - self.assertNotIn( - 'hello', Instance.objects.get(pk=dataid).tags.names()) + self.assertNotIn("hello", Instance.objects.get(pk=dataid).tags.names()) def test_labels_action_with_params(self): self._make_submissions() xform = XForm.objects.all()[0] pk = xform.id dataid = xform.instances.all()[0].id - view = DataViewSet.as_view({ - 'get': 'labels' - }) + view = DataViewSet.as_view({"get": "labels"}) - request = self.factory.get('/', **self.extra) - response = view(request, pk=pk, dataid=dataid, label='hello') + request = self.factory.get("/", **self.extra) + response = view(request, pk=pk, dataid=dataid, label="hello") self.assertEqual(response.status_code, 200) def test_data_list_filter_by_user(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk bobs_data = _data_list(formid)[0] previous_user = self.user - self._create_user_and_login('alice', 'alice') - self.assertEqual(self.user.username, 'alice') + self._create_user_and_login("alice", "alice") + self.assertEqual(self.user.username, "alice") self.assertNotEqual(previous_user, self.user) ReadOnlyRole.add(self.user, self.xform) @@ -1281,73 +1295,72 @@ def test_data_list_filter_by_user(self): # publish alice's form self._publish_transportation_form() - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} formid = self.xform.pk alice_data = _data_list(formid)[0] - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) # should be both bob's and alice's form self.assertEqual( - sorted(response.data, key=lambda x: x['id']), - sorted([bobs_data, alice_data], key=lambda x: x['id'])) + sorted(response.data, key=lambda x: x["id"]), + sorted([bobs_data, alice_data], key=lambda x: x["id"]), + ) # apply filter, see only bob's forms - request = self.factory.get('/', data={'owner': 'bob'}, **self.extra) + request = self.factory.get("/", data={"owner": "bob"}, **self.extra) response = view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, [bobs_data]) # apply filter, see only alice's forms - request = self.factory.get('/', data={'owner': 'alice'}, **self.extra) + request = self.factory.get("/", data={"owner": "alice"}, **self.extra) response = view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, [alice_data]) # apply filter, see a non existent user - request = self.factory.get('/', data={'owner': 'noone'}, **self.extra) + request = self.factory.get("/", data={"owner": "noone"}, **self.extra) response = view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) def test_get_enketo_edit_url(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'enketo'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "enketo"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) # add data check - self.assertEqual( - response.data, - {'detail': 'return_url not provided.'}) + self.assertEqual(response.data, {"detail": "return_url not provided."}) request = self.factory.get( - '/', - data={'return_url': "http://test.io/test_url"}, **self.extra) + "/", data={"return_url": "http://test.io/test_url"}, **self.extra + ) with HTTMock(enketo_edit_mock): response = view(request, pk=formid, dataid=dataid) self.assertEqual( - response.data['url'], + response.data["url"], "https://hmh2a.enketo.ona.io/edit/XA0bG8Df?instance_id=" "672927e3-9ad4-42bb-9538-388ea1fb6699&returnUrl=http://test" - ".io/test_url") + ".io/test_url", + ) with HTTMock(enketo_mock_http_413): response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) def test_get_form_public_data(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/') + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/") formid = self.xform.pk response = view(request, pk=formid) @@ -1362,25 +1375,26 @@ def test_get_form_public_data(self): self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) self.assertTrue(self.xform.instances.count()) - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk data = _data_instance(dataid) self.assertDictContainsSubset( - data, sorted(response.data, key=lambda x: x['_id'])[0]) + data, sorted(response.data, key=lambda x: x["_id"])[0] + ) # access to a public data as other user - self._create_user_and_login('alice', 'alice') - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} - request = self.factory.get('/', **self.extra) + self._create_user_and_login("alice", "alice") + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) self.assertTrue(self.xform.instances.count()) - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk data = _data_instance(dataid) self.assertDictContainsSubset( - data, sorted(response.data, key=lambda x: x['_id'])[0]) + data, sorted(response.data, key=lambda x: x["_id"])[0] + ) def test_same_submission_with_different_attachments(self): """ @@ -1394,90 +1408,104 @@ def test_same_submission_with_different_attachments(self): """ xform = self._publish_markdown(images_md, self.user) submission_file = NamedTemporaryFile(delete=False) - with open(submission_file.name, 'w') as xml_file: + with open(submission_file.name, "w") as xml_file: xml_file.write( "" "1335783522563.jpg" "1442323232322.jpg" "uuid:729f173c688e482486a48661700455ff" - "" % - (xform.id_string)) + "" % (xform.id_string) + ) media_file = "1335783522563.jpg" self._make_submission_w_attachment( submission_file.name, - os.path.join(self.this_directory, 'fixtures', 'transportation', - 'instances', self.surveys[0], media_file)) - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + media_file, + ), + ) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) formid = xform.pk data = { - 'id': formid, - 'id_string': xform.id_string, - 'title': xform.title, - 'description': '', - 'url': 'http://testserver/api/v1/data/%s' % formid + "id": formid, + "id_string": xform.id_string, + "title": xform.title, + "description": "", + "url": "http://testserver/api/v1/data/%s" % formid, } self.assertEqual(response.data[1], data) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - instance = xform.instances.all().order_by('id')[0] + instance = xform.instances.all().order_by("id")[0] dataid = instance.pk attachment = instance.attachments.all().first() data = { - '_bamboo_dataset_id': '', - '_attachments': [{ - 'download_url': get_attachment_url(attachment), - 'small_download_url': - get_attachment_url(attachment, 'small'), - 'medium_download_url': - get_attachment_url(attachment, 'medium'), - 'mimetype': attachment.mimetype, - 'instance': attachment.instance.pk, - 'filename': attachment.media_file.name, - 'name': attachment.name, - 'id': attachment.pk, - 'xform': xform.id} + "_bamboo_dataset_id": "", + "_attachments": [ + { + "download_url": get_attachment_url(attachment), + "small_download_url": get_attachment_url(attachment, "small"), + "medium_download_url": get_attachment_url(attachment, "medium"), + "mimetype": attachment.mimetype, + "instance": attachment.instance.pk, + "filename": attachment.media_file.name, + "name": attachment.name, + "id": attachment.pk, + "xform": xform.id, + } ], - '_geolocation': [None, None], - '_xform_id_string': xform.id_string, - '_status': 'submitted_via_web', - '_id': dataid, - 'image1': '1335783522563.jpg' + "_geolocation": [None, None], + "_xform_id_string": xform.id_string, + "_status": "submitted_via_web", + "_id": dataid, + "image1": "1335783522563.jpg", } self.assertDictContainsSubset(data, sorted(response.data)[0]) - patch_value = 'onadata.libs.utils.logger_tools.get_filtered_instances' + patch_value = "onadata.libs.utils.logger_tools.get_filtered_instances" with patch(patch_value) as get_filtered_instances: get_filtered_instances.return_value = Instance.objects.filter( - uuid='#doesnotexist') + uuid="#doesnotexist" + ) media_file = "1442323232322.jpg" self._make_submission_w_attachment( submission_file.name, - os.path.join(self.this_directory, 'fixtures', 'transportation', - 'instances', self.surveys[0], media_file)) + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + media_file, + ), + ) attachment = Attachment.objects.get(name=media_file) - data['_attachments'] = data.get('_attachments') + [{ - 'download_url': get_attachment_url(attachment), - 'small_download_url': - get_attachment_url(attachment, 'small'), - 'medium_download_url': - get_attachment_url(attachment, 'medium'), - 'mimetype': attachment.mimetype, - 'instance': attachment.instance.pk, - 'filename': attachment.media_file.name, - 'name': attachment.name, - 'id': attachment.pk, - 'xform': xform.id - }] + data["_attachments"] = data.get("_attachments") + [ + { + "download_url": get_attachment_url(attachment), + "small_download_url": get_attachment_url(attachment, "small"), + "medium_download_url": get_attachment_url(attachment, "medium"), + "mimetype": attachment.mimetype, + "instance": attachment.instance.pk, + "filename": attachment.media_file.name, + "name": attachment.name, + "id": attachment.pk, + "xform": xform.id, + } + ] self.maxDiff = None response = view(request, pk=formid) - self.assertDictContainsSubset(sorted([data])[0], - sorted(response.data)[0]) + self.assertDictContainsSubset(sorted([data])[0], sorted(response.data)[0]) self.assertEqual(response.status_code, 200) submission_file.close() os.unlink(submission_file.name) @@ -1485,8 +1513,8 @@ def test_same_submission_with_different_attachments(self): def test_data_w_attachment(self): self._submit_transport_instance_w_attachment() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) formid = self.xform.pk @@ -1496,63 +1524,60 @@ def test_data_w_attachment(self): self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) self.assertTrue(self.xform.instances.count()) - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk data = { - '_bamboo_dataset_id': '', - '_attachments': [{ - 'download_url': get_attachment_url(self.attachment), - 'small_download_url': - get_attachment_url(self.attachment, 'small'), - 'medium_download_url': - get_attachment_url(self.attachment, 'medium'), - 'mimetype': self.attachment.mimetype, - 'instance': self.attachment.instance.pk, - 'filename': self.attachment.media_file.name, - 'name': self.attachment.name, - 'id': self.attachment.pk, - 'xform': self.xform.id} + "_bamboo_dataset_id": "", + "_attachments": [ + { + "download_url": get_attachment_url(self.attachment), + "small_download_url": get_attachment_url(self.attachment, "small"), + "medium_download_url": get_attachment_url( + self.attachment, "medium" + ), + "mimetype": self.attachment.mimetype, + "instance": self.attachment.instance.pk, + "filename": self.attachment.media_file.name, + "name": self.attachment.name, + "id": self.attachment.pk, + "xform": self.xform.id, + } ], - '_geolocation': [None, None], - '_xform_id_string': 'transportation_2011_07_25', - 'transport/available_transportation_types_to_referral_facility': - 'none', - '_status': 'submitted_via_web', - '_id': dataid + "_geolocation": [None, None], + "_xform_id_string": "transportation_2011_07_25", + "transport/available_transportation_types_to_referral_facility": "none", + "_status": "submitted_via_web", + "_id": dataid, } self.assertDictContainsSubset(data, sorted(response.data)[0]) data = { - '_xform_id_string': 'transportation_2011_07_25', - 'transport/available_transportation_types_to_referral_facility': - 'none', - '_submitted_by': 'bob', + "_xform_id_string": "transportation_2011_07_25", + "transport/available_transportation_types_to_referral_facility": "none", + "_submitted_by": "bob", } - view = DataViewSet.as_view({'get': 'retrieve'}) + view = DataViewSet.as_view({"get": "retrieve"}) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, dict) self.assertDictContainsSubset(data, response.data) - @patch('onadata.apps.api.viewsets.data_viewset.send_message') + @patch("onadata.apps.api.viewsets.data_viewset.send_message") def test_delete_submission(self, send_message_mock): self._make_submissions() formid = self.xform.pk - dataid = self.xform.instances.all().order_by('id')[0].pk - view = DataViewSet.as_view({ - 'delete': 'destroy', - 'get': 'list' - }) + dataid = self.xform.instances.all().order_by("id")[0].pk + view = DataViewSet.as_view({"delete": "destroy", "get": "list"}) # get the first xform instance returned first_xform_instance = self.xform.instances.filter(pk=dataid) self.assertEqual(first_xform_instance[0].deleted_by, None) # 4 submissions - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(len(response.data), 4) - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 204) first_xform_instance = self.xform.instances.filter(pk=dataid) @@ -1560,65 +1585,66 @@ def test_delete_submission(self, send_message_mock): # message sent upon delete self.assertTrue(send_message_mock.called) send_message_mock.assert_called_with( - instance_id=dataid, target_id=formid, target_type=XFORM, - user=request.user, message_verb=SUBMISSION_DELETED) + instance_id=dataid, + target_id=formid, + target_type=XFORM, + user=request.user, + message_verb=SUBMISSION_DELETED, + ) # second delete of same submission should return 404 - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 404) # remaining 3 submissions - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(len(response.data), 3) - self._create_user_and_login(username='alice', password='alice') + self._create_user_and_login(username="alice", password="alice") # Managers can delete role.ManagerRole.add(self.user, self.xform) - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} - request = self.factory.delete('/', **self.extra) - dataid = self.xform.instances.filter(deleted_at=None)\ - .order_by('id')[0].pk + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} + request = self.factory.delete("/", **self.extra) + dataid = self.xform.instances.filter(deleted_at=None).order_by("id")[0].pk response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 204) # remaining 3 submissions - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(len(response.data), 2) - @patch('onadata.apps.api.viewsets.data_viewset.send_message') - @patch('onadata.apps.viewer.signals._post_process_submissions') - def test_post_save_signal_on_submission_deletion(self, mock, - send_message_mock): + @patch("onadata.apps.api.viewsets.data_viewset.send_message") + @patch("onadata.apps.viewer.signals._post_process_submissions") + def test_post_save_signal_on_submission_deletion(self, mock, send_message_mock): # test that post_save_submission signal is sent # create form xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/tutorial.xlsx" + "../fixtures/tutorial/tutorial.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) # create submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) self._make_submission(xml_submission_file_path) - view = DataViewSet.as_view({ - 'delete': 'destroy', - 'get': 'list' - }) + view = DataViewSet.as_view({"delete": "destroy", "get": "list"}) self.assertEqual(self.response.status_code, 201) formid = self.xform.pk - dataid = self.xform.instances.all().order_by('id')[0].pk - request = self.factory.delete('/', **self.extra) + dataid = self.xform.instances.all().order_by("id")[0].pk + request = self.factory.delete("/", **self.extra) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 204) @@ -1632,8 +1658,7 @@ def test_post_save_signal_on_submission_deletion(self, mock, self.assertEqual(mock.call_count, 1) self.assertTrue(send_message_mock.called) - @patch( - 'onadata.apps.api.viewsets.data_viewset.send_message') + @patch("onadata.apps.api.viewsets.data_viewset.send_message") def test_deletion_of_bulk_submissions(self, send_message_mock): self._make_submissions() @@ -1643,18 +1668,15 @@ def test_deletion_of_bulk_submissions(self, send_message_mock): self.assertEqual(initial_count, 4) self.assertEqual(self.xform.num_of_submissions, 4) - view = DataViewSet.as_view({'delete': 'destroy'}) + view = DataViewSet.as_view({"delete": "destroy"}) # test with invalid instance id's data = {"instance_ids": "john,doe"} - request = self.factory.delete('/', data=data, **self.extra) + request = self.factory.delete("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 400) - self.assertEqual( - response.data.get('detail'), - u"Invalid data ids were provided." - ) + self.assertEqual(response.data.get("detail"), "Invalid data ids were provided.") self.xform.refresh_from_db() current_count = self.xform.instances.filter(deleted_at=None).count() self.assertEqual(current_count, initial_count) @@ -1663,37 +1685,43 @@ def test_deletion_of_bulk_submissions(self, send_message_mock): # test with valid instance id's records_to_be_deleted = self.xform.instances.all()[:2] - instance_ids = ','.join([str(i.pk) for i in records_to_be_deleted]) + instance_ids = ",".join([str(i.pk) for i in records_to_be_deleted]) data = {"instance_ids": instance_ids} - request = self.factory.delete('/', data=data, **self.extra) + request = self.factory.delete("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual( - response.data.get('message'), - "%d records were deleted" % len(records_to_be_deleted) + response.data.get("message"), + "%d records were deleted" % len(records_to_be_deleted), ) self.assertTrue(send_message_mock.called) send_message_mock.called_with( - [str(i.pk) for i in records_to_be_deleted], formid, XFORM, - request.user, SUBMISSION_DELETED) + [str(i.pk) for i in records_to_be_deleted], + formid, + XFORM, + request.user, + SUBMISSION_DELETED, + ) self.xform.refresh_from_db() current_count = self.xform.instances.filter(deleted_at=None).count() self.assertNotEqual(current_count, initial_count) self.assertEqual(current_count, 2) self.assertEqual(self.xform.num_of_submissions, 2) - @patch('onadata.apps.api.viewsets.data_viewset.send_message') + @patch("onadata.apps.api.viewsets.data_viewset.send_message") def test_delete_submission_inactive_form(self, send_message_mock): self._make_submissions() formid = self.xform.pk - dataid = self.xform.instances.all().order_by('id')[0].pk - view = DataViewSet.as_view({ - 'delete': 'destroy', - }) + dataid = self.xform.instances.all().order_by("id")[0].pk + view = DataViewSet.as_view( + { + "delete": "destroy", + } + ) - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 204) @@ -1702,27 +1730,34 @@ def test_delete_submission_inactive_form(self, send_message_mock): self.xform.downloadable = False self.xform.save() - dataid = self.xform.instances.filter(deleted_at=None)\ - .order_by('id')[0].pk + dataid = self.xform.instances.filter(deleted_at=None).order_by("id")[0].pk - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 400) self.assertTrue(send_message_mock.called) - @patch('onadata.apps.api.viewsets.data_viewset.send_message') + @patch("onadata.apps.api.viewsets.data_viewset.send_message") def test_delete_submissions(self, send_message_mock): xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/tutorial.xlsx" + "../fixtures/tutorial/tutorial.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) # Add multiple submissions for x in range(1, 11): path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - 'tutorial', 'instances', 'uuid{}'.format(x), 'submission.xml') + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial", + "instances", + "uuid{}".format(x), + "submission.xml", + ) self._make_submission(path) x += 1 self.xform.refresh_from_db() @@ -1731,23 +1766,27 @@ def test_delete_submissions(self, send_message_mock): self.assertEqual(initial_count, 9) self.assertEqual(self.xform.num_of_submissions, 9) - view = DataViewSet.as_view({'delete': 'destroy'}) + view = DataViewSet.as_view({"delete": "destroy"}) deleted_instances_subset = self.xform.instances.all()[:6] - instance_ids = ','.join([str(i.pk) for i in deleted_instances_subset]) - data = {"instance_ids": instance_ids, 'delete_all': False} + instance_ids = ",".join([str(i.pk) for i in deleted_instances_subset]) + data = {"instance_ids": instance_ids, "delete_all": False} - request = self.factory.delete('/', data=data, **self.extra) + request = self.factory.delete("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual( - response.data.get('message'), - "%d records were deleted" % len(deleted_instances_subset) + response.data.get("message"), + "%d records were deleted" % len(deleted_instances_subset), ) self.assertTrue(send_message_mock.called) send_message_mock.called_with( - [str(i.pk) for i in deleted_instances_subset], formid, XFORM, - request.user, SUBMISSION_DELETED) + [str(i.pk) for i in deleted_instances_subset], + formid, + XFORM, + request.user, + SUBMISSION_DELETED, + ) # Test that num of submissions for the form is successfully updated self.xform.refresh_from_db() @@ -1757,92 +1796,80 @@ def test_delete_submissions(self, send_message_mock): self.assertEqual(self.xform.num_of_submissions, 3) # Test delete_all submissions for the form - data = {'delete_all': True} - request = self.factory.delete('/', data=data, **self.extra) + data = {"delete_all": True} + request = self.factory.delete("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - self.assertEqual( - response.data.get('message'), - "3 records were deleted") + self.assertEqual(response.data.get("message"), "3 records were deleted") # Test project details updated successfully self.assertEqual( self.xform.project.date_modified.strftime("%Y-%m-%d %H:%M:%S"), - timezone.now().strftime("%Y-%m-%d %H:%M:%S")) + timezone.now().strftime("%Y-%m-%d %H:%M:%S"), + ) # Test XForm now contains no submissions self.xform.refresh_from_db() - delete_all_current_count = self.xform.instances.filter( - deleted_at=None).count() + delete_all_current_count = self.xform.instances.filter(deleted_at=None).count() self.assertNotEqual(current_count, delete_all_current_count) self.assertEqual(delete_all_current_count, 0) self.assertEqual(self.xform.num_of_submissions, 0) - @patch('onadata.apps.api.viewsets.data_viewset.send_message') + @patch("onadata.apps.api.viewsets.data_viewset.send_message") def test_delete_submission_by_editor(self, send_message_mock): self._make_submissions() formid = self.xform.pk - dataid = self.xform.instances.all().order_by('id')[0].pk - view = DataViewSet.as_view({ - 'delete': 'destroy', - 'get': 'list' - }) + dataid = self.xform.instances.all().order_by("id")[0].pk + view = DataViewSet.as_view({"delete": "destroy", "get": "list"}) # 4 submissions - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(len(response.data), 4) - self._create_user_and_login(username='alice', password='alice') + self._create_user_and_login(username="alice", password="alice") # Editor can delete submission role.EditorRole.add(self.user, self.xform) - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} - request = self.factory.delete('/', **self.extra) - dataid = self.xform.instances.filter(deleted_at=None)\ - .order_by('id')[0].pk + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} + request = self.factory.delete("/", **self.extra) + dataid = self.xform.instances.filter(deleted_at=None).order_by("id")[0].pk response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 204) # remaining 3 submissions - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(len(response.data), 3) self.assertTrue(send_message_mock.called) - @patch('onadata.apps.api.viewsets.data_viewset.send_message') + @patch("onadata.apps.api.viewsets.data_viewset.send_message") def test_delete_submission_by_owner(self, send_message_mock): self._make_submissions() formid = self.xform.pk - dataid = self.xform.instances.all().order_by('id')[0].pk - view = DataViewSet.as_view({ - 'delete': 'destroy', - 'get': 'list' - }) + dataid = self.xform.instances.all().order_by("id")[0].pk + view = DataViewSet.as_view({"delete": "destroy", "get": "list"}) # 4 submissions - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(len(response.data), 4) - self._create_user_and_login(username='alice', password='alice') + self._create_user_and_login(username="alice", password="alice") # Owner can delete submission role.OwnerRole.add(self.user, self.xform) - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} - request = self.factory.delete('/', **self.extra) - dataid = self.xform.instances.filter(deleted_at=None)\ - .order_by('id')[0].pk + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} + request = self.factory.delete("/", **self.extra) + dataid = self.xform.instances.filter(deleted_at=None).order_by("id")[0].pk response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 204) # remaining 3 submissions - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(len(response.data), 3) self.assertTrue(send_message_mock.called) @@ -1850,142 +1877,110 @@ def test_delete_submission_by_owner(self, send_message_mock): def test_geojson_format(self): self._publish_submit_geojson() - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk - view = DataViewSet.as_view({'get': 'retrieve'}) - data_get = { - "fields": 'today' - } - request = self.factory.get('/', data=data_get, **self.extra) - response = view(request, pk=self.xform.pk, dataid=dataid, - format='geojson') + view = DataViewSet.as_view({"get": "retrieve"}) + data_get = {"fields": "today"} + request = self.factory.get("/", data=data_get, **self.extra) + response = view(request, pk=self.xform.pk, dataid=dataid, format="geojson") self.assertEqual(response.status_code, 200) self.assertEqual(self.xform.instances.count(), 4) test_geo = { - 'type': 'Feature', - 'geometry': { - 'type': 'GeometryCollection', - 'geometries': [{ - 'type': 'Point', - 'coordinates': [ - 36.787219, - -1.294197 - ] - } - ] + "type": "Feature", + "geometry": { + "type": "GeometryCollection", + "geometries": [ + {"type": "Point", "coordinates": [36.787219, -1.294197]} + ], }, - 'properties': { - 'id': dataid, - 'xform': self.xform.pk, - 'today': '2015-01-15' - } + "properties": {"id": dataid, "xform": self.xform.pk, "today": "2015-01-15"}, } self.assertEqual(response.data, test_geo) - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', data=data_get, **self.extra) - response = view(request, pk=self.xform.pk, format='geojson') - instances = self.xform.instances.all().order_by('id') + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", data=data_get, **self.extra) + response = view(request, pk=self.xform.pk, format="geojson") + instances = self.xform.instances.all().order_by("id") data = { - 'type': 'FeatureCollection', - 'features': [ + "type": "FeatureCollection", + "features": [ { - 'type': 'Feature', - 'geometry': { - 'type': 'GeometryCollection', - 'geometries': [{ - 'type': 'Point', - 'coordinates': [ - 36.787219, - -1.294197 - ] - } - ] + "type": "Feature", + "geometry": { + "type": "GeometryCollection", + "geometries": [ + {"type": "Point", "coordinates": [36.787219, -1.294197]} + ], + }, + "properties": { + "id": instances[0].pk, + "xform": self.xform.pk, + "today": "2015-01-15", }, - 'properties': { - 'id': instances[0].pk, - 'xform': self.xform.pk, - 'today': '2015-01-15' - } }, { - 'type': 'Feature', - 'geometry': { - 'type': 'GeometryCollection', - 'geometries': [{ - 'type': 'Point', - 'coordinates': [ - 36.787219, - -1.294197 - ] - } - ] + "type": "Feature", + "geometry": { + "type": "GeometryCollection", + "geometries": [ + {"type": "Point", "coordinates": [36.787219, -1.294197]} + ], + }, + "properties": { + "id": instances[1].pk, + "xform": self.xform.pk, + "today": "2015-01-15", }, - 'properties': { - 'id': instances[1].pk, - 'xform': self.xform.pk, - 'today': '2015-01-15' - } }, { - 'type': 'Feature', - 'geometry': { - 'type': 'GeometryCollection', - 'geometries': [{ - 'type': 'Point', - 'coordinates': [ - 36.787219, - -1.294197 - ] - } - ] + "type": "Feature", + "geometry": { + "type": "GeometryCollection", + "geometries": [ + {"type": "Point", "coordinates": [36.787219, -1.294197]} + ], + }, + "properties": { + "id": instances[2].pk, + "xform": self.xform.pk, + "today": "2015-01-15", }, - 'properties': { - 'id': instances[2].pk, - 'xform': self.xform.pk, - 'today': '2015-01-15' - } }, { - 'type': 'Feature', - 'geometry': { - 'type': 'GeometryCollection', - 'geometries': [{ - 'type': 'Point', - 'coordinates': [ - 36.787219, - -1.294197 - ] - } - ] + "type": "Feature", + "geometry": { + "type": "GeometryCollection", + "geometries": [ + {"type": "Point", "coordinates": [36.787219, -1.294197]} + ], }, - 'properties': { - 'id': instances[3].pk, - 'xform': self.xform.pk, - 'today': '2015-01-15' - } - } - ] + "properties": { + "id": instances[3].pk, + "xform": self.xform.pk, + "today": "2015-01-15", + }, + }, + ], } self.assertEqual(response.status_code, 200) self.assertEqual(response.data, data) - @patch( - 'onadata.apps.api.viewsets.data_viewset' - '.DataViewSet.paginate_queryset') + @patch("onadata.apps.api.viewsets.data_viewset" ".DataViewSet.paginate_queryset") def test_retry_on_operational_error(self, mock_paginate_queryset): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk mock_paginate_queryset.side_effect = [ OperationalError, - Instance.objects.filter(xform_id=formid)[:2]] + Instance.objects.filter(xform_id=formid)[:2], + ] - request = self.factory.get('/', data={"page": "1", "page_size": 2}, - **self.extra) + request = self.factory.get( + "/", data={"page": "1", "page_size": 2}, **self.extra + ) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(mock_paginate_queryset.call_count, 2) @@ -1993,137 +1988,118 @@ def test_retry_on_operational_error(self, mock_paginate_queryset): def test_geojson_geofield(self): self._publish_submit_geojson() - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk - data_get = { - "geo_field": 'location', - "fields": 'today' - } + data_get = {"geo_field": "location", "fields": "today"} - view = DataViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/', data=data_get, **self.extra) - response = view(request, pk=self.xform.pk, dataid=dataid, - format='geojson') + view = DataViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", data=data_get, **self.extra) + response = view(request, pk=self.xform.pk, dataid=dataid, format="geojson") self.assertEqual(response.status_code, 200) test_loc = geojson.Feature( - geometry=geojson.GeometryCollection([ - geojson.Point((36.787219, -1.294197))]), - properties={ - 'xform': self.xform.pk, - 'id': dataid, - 'today': '2015-01-15' - } + geometry=geojson.GeometryCollection( + [geojson.Point((36.787219, -1.294197))] + ), + properties={"xform": self.xform.pk, "id": dataid, "today": "2015-01-15"}, ) - if 'id' in test_loc: - test_loc.pop('id') + if "id" in test_loc: + test_loc.pop("id") self.assertEqual(response.data, test_loc) - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) - request = self.factory.get('/', data=data_get, **self.extra) - response = view(request, pk=self.xform.pk, format='geojson') + request = self.factory.get("/", data=data_get, **self.extra) + response = view(request, pk=self.xform.pk, format="geojson") self.assertEqual(response.status_code, 200) - self.assertEquals(response.data['type'], 'FeatureCollection') - self.assertEquals(len(response.data['features']), 4) - self.assertEquals(response.data['features'][0]['type'], 'Feature') - self.assertEquals( - response.data['features'][0]['geometry']['geometries'][0]['type'], - 'Point' + self.assertEqual(response.data["type"], "FeatureCollection") + self.assertEqual(len(response.data["features"]), 4) + self.assertEqual(response.data["features"][0]["type"], "Feature") + self.assertEqual( + response.data["features"][0]["geometry"]["geometries"][0]["type"], "Point" ) def test_geojson_linestring(self): self._publish_submit_geojson() - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk - data_get = { - "geo_field": 'path', - "fields": 'today,path' - } + data_get = {"geo_field": "path", "fields": "today,path"} - view = DataViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/', data=data_get, **self.extra) - response = view(request, pk=self.xform.pk, dataid=dataid, - format='geojson') + view = DataViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", data=data_get, **self.extra) + response = view(request, pk=self.xform.pk, dataid=dataid, format="geojson") self.assertEqual(response.status_code, 200) - self.assertEquals(response.data['type'], 'Feature') - self.assertEquals(len(response.data['geometry']['coordinates']), 5) - self.assertIn('path', response.data['properties']) - self.assertEquals(response.data['geometry']['type'], 'LineString') + self.assertEqual(response.data["type"], "Feature") + self.assertEqual(len(response.data["geometry"]["coordinates"]), 5) + self.assertIn("path", response.data["properties"]) + self.assertEqual(response.data["geometry"]["type"], "LineString") - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) - request = self.factory.get('/', data=data_get, **self.extra) - response = view(request, pk=self.xform.pk, format='geojson') + request = self.factory.get("/", data=data_get, **self.extra) + response = view(request, pk=self.xform.pk, format="geojson") self.assertEqual(response.status_code, 200) - self.assertEquals(response.data['type'], 'FeatureCollection') - self.assertEquals(len(response.data['features']), 4) - self.assertEquals(response.data['features'][0]['type'], 'Feature') - self.assertEquals(response.data['features'][0]['geometry']['type'], - 'LineString') + self.assertEqual(response.data["type"], "FeatureCollection") + self.assertEqual(len(response.data["features"]), 4) + self.assertEqual(response.data["features"][0]["type"], "Feature") + self.assertEqual(response.data["features"][0]["geometry"]["type"], "LineString") def test_geojson_polygon(self): self._publish_submit_geojson() - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk - data_get = { - "geo_field": 'shape', - "fields": 'today,shape' - } + data_get = {"geo_field": "shape", "fields": "today,shape"} - view = DataViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/', data=data_get, **self.extra) - response = view(request, pk=self.xform.pk, dataid=dataid, - format='geojson') + view = DataViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", data=data_get, **self.extra) + response = view(request, pk=self.xform.pk, dataid=dataid, format="geojson") self.assertEqual(response.status_code, 200) - self.assertEquals(response.data['type'], 'Feature') - self.assertEquals(len(response.data['geometry']['coordinates'][0]), 6) - self.assertIn('shape', response.data['properties']) - self.assertEquals(response.data['geometry']['type'], 'Polygon') + self.assertEqual(response.data["type"], "Feature") + self.assertEqual(len(response.data["geometry"]["coordinates"][0]), 6) + self.assertIn("shape", response.data["properties"]) + self.assertEqual(response.data["geometry"]["type"], "Polygon") - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) - request = self.factory.get('/', data=data_get, **self.extra) - response = view(request, pk=self.xform.pk, format='geojson') + request = self.factory.get("/", data=data_get, **self.extra) + response = view(request, pk=self.xform.pk, format="geojson") self.assertEqual(response.status_code, 200) - self.assertEquals(response.data['type'], 'FeatureCollection') - self.assertEquals(len(response.data['features']), 4) - self.assertEquals(response.data['features'][0]['type'], 'Feature') - self.assertEquals(response.data['features'][0]['geometry']['type'], - 'Polygon') + self.assertEqual(response.data["type"], "FeatureCollection") + self.assertEqual(len(response.data["features"]), 4) + self.assertEqual(response.data["features"][0]["type"], "Feature") + self.assertEqual(response.data["features"][0]["geometry"]["type"], "Polygon") def test_data_in_public_project(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk with HTTMock(enketo_urls_mock): response = view(request, pk=formid) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) # get project id projectid = self.xform.project.pk - view = ProjectViewSet.as_view({ - 'put': 'update' - }) + view = ProjectViewSet.as_view({"put": "update"}) - data = {'public': True, - 'name': 'test project', - 'owner': 'http://testserver/api/v1/users/%s' - % self.user.username} - request = self.factory.put('/', data=data, **self.extra) + data = { + "public": True, + "name": "test project", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + } + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 200) @@ -2131,12 +2107,12 @@ def test_data_in_public_project(self): self.assertEqual(self.xform.shared, True) # anonymous user - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/') + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/") formid = self.xform.pk response = view(request, pk=formid) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) def test_data_diff_version(self): @@ -2146,85 +2122,96 @@ def test_data_diff_version(self): self.xform.save() # make more submission after form update - surveys = ['transport_2011-07-25_19-05-36-edited'] + surveys = ["transport_2011-07-25_19-05-36-edited"] main_directory = os.path.dirname(main_tests.__file__) - paths = [os.path.join(main_directory, 'fixtures', 'transportation', - 'instances_w_uuid', s, s + '.xml') - for s in surveys] + paths = [ + os.path.join( + main_directory, + "fixtures", + "transportation", + "instances_w_uuid", + s, + s + ".xml", + ) + for s in surveys + ] - auth = DigestAuth('bob', 'bob') + auth = DigestAuth("bob", "bob") for path in paths: self._make_submission(path, None, None, auth=auth) - data_view = DataViewSet.as_view({'get': 'list'}) + data_view = DataViewSet.as_view({"get": "list"}) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = data_view(request, pk=self.xform.pk) - self.assertEquals(len(response.data), 5) + self.assertEqual(len(response.data), 5) query_str = '{"_version": "2014111"}' - request = self.factory.get('/?query=%s' % query_str, **self.extra) + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = data_view(request, pk=self.xform.pk) - self.assertEquals(len(response.data), 4) + self.assertEqual(len(response.data), 4) query_str = '{"_version": "212121211"}' - request = self.factory.get('/?query=%s' % query_str, **self.extra) + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = data_view(request, pk=self.xform.pk) - self.assertEquals(len(response.data), 1) + self.assertEqual(len(response.data), 1) def test_last_modified_on_data_list_response(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk response = view(request, pk=formid) - self.assertEquals(response.status_code, 200) - self.assertEqual(response.get('Cache-Control'), 'max-age=60') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.get("Cache-Control"), "max-age=60") - self.assertTrue(response.has_header('ETag')) - etag_value = response.get('ETag') + self.assertTrue(response.has_header("ETag")) + etag_value = response.get("ETag") - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk response = view(request, pk=formid) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) - self.assertEquals(etag_value, response.get('ETag')) + self.assertEqual(etag_value, response.get("ETag")) # delete one submission inst = Instance.objects.filter(xform=self.xform) inst[0].delete() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk response = view(request, pk=formid) - self.assertEquals(response.status_code, 200) - self.assertNotEquals(etag_value, response.get('ETag')) + self.assertEqual(response.status_code, 200) + self.assertNotEqual(etag_value, response.get("ETag")) def test_submission_history(self): """Test submission json includes has_history key""" # create form xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/tutorial.xlsx" + "../fixtures/tutorial/tutorial.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) # create submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) self._make_submission(xml_submission_file_path) @@ -2235,27 +2222,32 @@ def test_submission_history(self): # edit submission xml_edit_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml", ) xml_edit_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml", ) client = DigestClient() - client.set_authorization('bob', 'bob', 'Digest') + client.set_authorization("bob", "bob", "Digest") self._make_submission(xml_edit_submission_file_path, client=client) self.assertEqual(self.response.status_code, 201) self.assertEqual(instance_count, Instance.objects.count()) - self.assertEqual(instance_history_count + 1, - InstanceHistory.objects.count()) + self.assertEqual(instance_history_count + 1, InstanceHistory.objects.count()) # retrieve submission history - view = DataViewSet.as_view({'get': 'history'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "history"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.xform.pk, dataid=instance.id) self.assertEqual(response.status_code, 200) @@ -2268,12 +2260,11 @@ def test_submission_history(self): def test_submission_history_not_digit(self): """Test submission json includes has_history key""" # retrieve submission history - view = DataViewSet.as_view({'get': 'history'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "history"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.xform.pk, dataid="boo!") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data['detail'], - 'Data ID should be an integer') + self.assertEqual(response.data["detail"], "Data ID should be an integer") history_instance_count = InstanceHistory.objects.count() self.assertEqual(history_instance_count, 0) @@ -2283,13 +2274,13 @@ def test_data_endpoint_etag_on_submission_edit(self): # create form xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/tutorial.xlsx" + "../fixtures/tutorial/tutorial.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) def _data_response(): - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.xform.pk) self.assertEqual(response.status_code, 200) @@ -2300,45 +2291,54 @@ def _data_response(): # create submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) self._make_submission(xml_submission_file_path) response = _data_response() - etag_data = response['Etag'] + etag_data = response["Etag"] # edit submission xml_edit_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml", ) client = DigestClient() - client.set_authorization('bob', 'bob', 'Digest') + client.set_authorization("bob", "bob", "Digest") self._make_submission(xml_edit_submission_file_path, client=client) self.assertEqual(self.response.status_code, 201) response = _data_response() - self.assertNotEqual(etag_data, response['Etag']) - etag_data = response['Etag'] + self.assertNotEqual(etag_data, response["Etag"]) + etag_data = response["Etag"] response = _data_response() - self.assertEqual(etag_data, response['Etag']) + self.assertEqual(etag_data, response["Etag"]) def test_submission_edit_w_blank_field(self): """Test submission json includes has_history key""" # create form xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/tutorial.xlsx" + "../fixtures/tutorial/tutorial.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) # create submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) self._make_submission(xml_submission_file_path) @@ -2348,11 +2348,14 @@ def test_submission_edit_w_blank_field(self): # edit submission xml_edit_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid_edited_blank.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited_blank.xml", ) client = DigestClient() - client.set_authorization('bob', 'bob', 'Digest') + client.set_authorization("bob", "bob", "Digest") self._make_submission(xml_edit_submission_file_path, client=client) self.assertEqual(self.response.status_code, 201) @@ -2360,12 +2363,12 @@ def test_submission_edit_w_blank_field(self): self.assertEqual(instance_count, Instance.objects.count()) # retrieve submission history - view = DataViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.xform.pk, dataid=instance.id) self.assertEqual(instance_count, Instance.objects.count()) - self.assertNotIn('name', response.data) + self.assertNotIn("name", response.data) def test_filterset(self): # create submissions to test with @@ -2374,9 +2377,9 @@ def test_filterset(self): # the original count of Instance objects instance_count = Instance.objects.all().count() # ## Test no filters - request = self.factory.get('/', **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + request = self.factory.get("/", **self.extra) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual(len(response.data), instance_count) # ## Test version # all the instances created have no version @@ -2385,23 +2388,25 @@ def test_filterset(self): instance = Instance.objects.last() instance.version = 777 instance.save() - request = self.factory.get('/', {'version': 777}, **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') - self.assertEqual(len(response.data), - Instance.objects.filter(version=777).count()) + request = self.factory.get("/", {"version": 777}, **self.extra) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") + self.assertEqual( + len(response.data), Instance.objects.filter(version=777).count() + ) # ## Test Status # all the instanced created have the same status i.e. # 'submitted_via_web' . We now set one instance to have a different # status and filter for it instance = Instance.objects.last() - instance.status = 'fortytwo' + instance.status = "fortytwo" instance.save() - request = self.factory.get('/', {'status': 'fortytwo'}, **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') - self.assertEqual(len(response.data), - Instance.objects.filter(status='fortytwo').count()) + request = self.factory.get("/", {"status": "fortytwo"}, **self.extra) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") + self.assertEqual( + len(response.data), Instance.objects.filter(status="fortytwo").count() + ) # ## Test date_created # all the instances created have the same date_created i.e. the # datetime at the time of creation @@ -2411,15 +2416,14 @@ def test_filterset(self): initial_year = instance.date_created.year instance.date_created = instance.date_created - one_year instance.save() - request = self.factory.get('/', - {'date_created__year__lt': initial_year}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + request = self.factory.get( + "/", {"date_created__year__lt": initial_year}, **self.extra + ) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual( len(response.data), - Instance.objects.filter( - date_created__year__lt=initial_year).count() + Instance.objects.filter(date_created__year__lt=initial_year).count(), ) # ## Test last_edited # all the instances created have None as the last_edited @@ -2427,73 +2431,66 @@ def test_filterset(self): # we can filter for this one instance instance.last_edited = instance.date_created - one_year instance.save() - request = self.factory.get('/', - {'last_edited__year': initial_year}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + request = self.factory.get( + "/", {"last_edited__year": initial_year}, **self.extra + ) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual( len(response.data), - Instance.objects.filter( - last_edited__year=initial_year).count() + Instance.objects.filter(last_edited__year=initial_year).count(), ) # ## Test uuid # all the instances created have different uuid values # we test this by looking for just one match - request = self.factory.get('/', - {'uuid': instance.uuid}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + request = self.factory.get("/", {"uuid": instance.uuid}, **self.extra) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual( - len(response.data), - Instance.objects.filter(uuid=instance.uuid).count() + len(response.data), Instance.objects.filter(uuid=instance.uuid).count() ) # ## Test user # all the forms are owned by a user named bob # we create a user named alice and confirm that data filtered for her # has a count fo 0 - user_alice = self._create_user('alice', 'alice') - request = self.factory.get('/', - {'user__id': user_alice.id}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + user_alice = self._create_user("alice", "alice") + request = self.factory.get("/", {"user__id": user_alice.id}, **self.extra) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual(len(response.data), 0) # we make one instance belong to user_alice and then filter for that instance.user = user_alice instance.save() - request = self.factory.get('/', - {'user__username': user_alice.username}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + request = self.factory.get( + "/", {"user__username": user_alice.username}, **self.extra + ) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual( len(response.data), - Instance.objects.filter(user__username=user_alice.username).count() + Instance.objects.filter(user__username=user_alice.username).count(), ) # ## Test submitted_by # submitted_by is mapped to the user field # to test, we do the same as we did for user above - user_mosh = self._create_user('mosh', 'mosh') - request = self.factory.get('/', - {'submitted_by__id': user_mosh.id}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + user_mosh = self._create_user("mosh", "mosh") + request = self.factory.get( + "/", {"submitted_by__id": user_mosh.id}, **self.extra + ) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual(len(response.data), 0) # we make one instance belong to user_mosh and then filter for that instance.user = user_mosh instance.save() - request = self.factory.get('/', - {'submitted_by__username': - user_mosh.username}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + request = self.factory.get( + "/", {"submitted_by__username": user_mosh.username}, **self.extra + ) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual( len(response.data), - Instance.objects.filter(user__username=user_mosh.username).count() + Instance.objects.filter(user__username=user_mosh.username).count(), ) # ## Test survey_type # all the instances created have the same survey_type @@ -2501,32 +2498,27 @@ def test_filterset(self): new_survey_type = SurveyType.objects.create(slug="hunter2") instance.survey_type = new_survey_type instance.save() - request = self.factory.get('/', - {'survey_type__slug': new_survey_type.slug}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + request = self.factory.get( + "/", {"survey_type__slug": new_survey_type.slug}, **self.extra + ) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual( len(response.data), - Instance.objects.filter( - survey_type__slug=new_survey_type.slug).count() + Instance.objects.filter(survey_type__slug=new_survey_type.slug).count(), ) # ## Test all_media_received # all the instances have media_all_received == True - request = self.factory.get('/', - {'media_all_received': 'false'}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + request = self.factory.get("/", {"media_all_received": "false"}, **self.extra) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual(len(response.data), 1) # we set one to False and filter for it instance.media_all_received = False instance.save() - request = self.factory.get('/', - {'media_all_received': 'false'}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + request = self.factory.get("/", {"media_all_received": "false"}, **self.extra) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual(len(response.data), 2) def test_floip_format(self): @@ -2534,26 +2526,27 @@ def test_floip_format(self): Test FLOIP output results. """ self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk request = self.factory.get( - '/', - HTTP_ACCEPT='application/vnd.org.flowinterop.results+json', - **self.extra) + "/", + HTTP_ACCEPT="application/vnd.org.flowinterop.results+json", + **self.extra, + ) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) response.render() floip_list = json.loads(response.content) self.assertTrue(isinstance(floip_list, list)) - floip_row = [x for x in floip_list if x[-2] == 'none'][0] - self.assertEqual(floip_row[0], - response.data[0]['_submission_time'] + '+00:00') - self.assertEqual(floip_row[2], 'bob') - self.assertEqual(floip_row[3], response.data[0]['_uuid']) + floip_row = [x for x in floip_list if x[-2] == "none"][0] + self.assertEqual(floip_row[0], response.data[0]["_submission_time"] + "+00:00") + self.assertEqual(floip_row[2], "bob") + self.assertEqual(floip_row[3], response.data[0]["_uuid"]) self.assertEqual( floip_row[4], - 'transport/available_transportation_types_to_referral_facility') - self.assertEqual(floip_row[5], 'none') + "transport/available_transportation_types_to_referral_facility", + ) + self.assertEqual(floip_row[5], "none") def test_submission_count_for_day_tracking(self): """ @@ -2567,8 +2560,7 @@ def test_submission_count_for_day_tracking(self): c_date = datetime.datetime.today() instances = Instance.objects.filter(date_created__date=c_date.date()) current_count = instances.count() - self.assertEqual( - form.submission_count_for_today, current_count) + self.assertEqual(form.submission_count_for_today, current_count) # Confirm that the submission count is decreased # accordingly @@ -2578,8 +2570,7 @@ def test_submission_count_for_day_tracking(self): inst_one.set_deleted() current_count -= 1 - self.assertEqual( - form.submission_count_for_today, current_count) + self.assertEqual(form.submission_count_for_today, current_count) # Confirm submission count isn't decreased if the # date_created is different @@ -2587,9 +2578,7 @@ def test_submission_count_for_day_tracking(self): inst_two.date_created = future_date inst_two.save() inst_two.set_deleted() - self.assertEqual( - form.submission_count_for_today, current_count - ) + self.assertEqual(form.submission_count_for_today, current_count) # Check that deletes made with no current count cached # are successful @@ -2602,22 +2591,22 @@ def test_data_query_multiple_condition(self): data """ self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) query_str = ( '{"$or":[{"transport/loop_over_transport_types_frequency' '/ambulance/frequency_to_referral_facility":"weekly","t' - 'ransport/loop_over_transport_types_frequency/ambulanc' + "ransport/loop_over_transport_types_frequency/ambulanc" 'e/frequency_to_referral_facility":"daily"}]}' ) - request = self.factory.get(f'/?query={query_str}', **self.extra) + request = self.factory.get(f"/?query={query_str}", **self.extra) response = view(request, pk=self.xform.pk) count = 0 for inst in self.xform.instances.all(): if inst.json.get( - 'transport/loop_over_transport_types_frequency' - '/ambulance/frequency_to_referral_facility' - ) in ['daily', 'weekly']: + "transport/loop_over_transport_types_frequency" + "/ambulance/frequency_to_referral_facility" + ) in ["daily", "weekly"]: count += 1 self.assertEqual(response.status_code, 200) @@ -2626,10 +2615,10 @@ def test_data_query_multiple_condition(self): query_str = ( '{"$or":[{"transport/loop_over_transport_types_frequency' '/ambulance/frequency_to_referral_facility":"weekly"}, {"t' - 'ransport/loop_over_transport_types_frequency/ambulanc' + "ransport/loop_over_transport_types_frequency/ambulanc" 'e/frequency_to_referral_facility":"daily"}]}' ) - request = self.factory.get(f'/?query={query_str}', **self.extra) + request = self.factory.get(f"/?query={query_str}", **self.extra) response = view(request, pk=self.xform.pk) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), count) @@ -2640,32 +2629,29 @@ def test_data_query_ornull(self): $or filter option """ self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) - query_str = ('{"$or": [{"_review_status":"3"}' - ', {"_review_status": null }]}') - request = self.factory.get('/?query=%s' % query_str, **self.extra) + query_str = '{"$or": [{"_review_status":"3"}' ', {"_review_status": null }]}' + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) - instances = self.xform.instances.all().order_by('pk') + instances = self.xform.instances.all().order_by("pk") instance = instances[0] self.assertFalse(instance.has_a_review) # Review instance - data = { - "instance": instance.id, - "status": SubmissionReview.APPROVED - } + data = {"instance": instance.id, "status": SubmissionReview.APPROVED} - serializer_instance = SubmissionReviewSerializer(data=data, context={ - "request": request}) + serializer_instance = SubmissionReviewSerializer( + data=data, context={"request": request} + ) serializer_instance.is_valid() serializer_instance.save() instance.refresh_from_db() @@ -2677,13 +2663,11 @@ def test_data_query_ornull(self): self.assertEqual(len(response.data), 3) # Switch review to pending - data = { - "instance": instance.id, - "status": SubmissionReview.PENDING - } + data = {"instance": instance.id, "status": SubmissionReview.PENDING} - serializer_instance = SubmissionReviewSerializer(data=data, context={ - "request": request}) + serializer_instance = SubmissionReviewSerializer( + data=data, context={"request": request} + ) serializer_instance.is_valid() serializer_instance.save() instance.refresh_from_db() @@ -2698,19 +2682,22 @@ def test_data_query_ornull(self): data = { "instance": instance.id, "status": SubmissionReview.REJECTED, - "note": "Testing" + "note": "Testing", } - serializer_instance = SubmissionReviewSerializer(data=data, context={ - "request": request}) + serializer_instance = SubmissionReviewSerializer( + data=data, context={"request": request} + ) serializer_instance.is_valid() serializer_instance.save() instance.refresh_from_db() # Assert ornull operator still works with multiple values - query_str = ('{"$or": [{"_review_status":"3"},' - ' {"_review_status": "2"}, {"_review_status": null}]}') - request = self.factory.get('/?query=%s' % query_str, **self.extra) + query_str = ( + '{"$or": [{"_review_status":"3"},' + ' {"_review_status": "2"}, {"_review_status": null}]}' + ) + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) @@ -2719,27 +2706,38 @@ def test_data_list_xml_format(self): """Test DataViewSet list XML""" # create submission media_file = "1335783522563.jpg" - self._make_submission_w_attachment(os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', 'transport_2011-07-25_19-05-49_2', - 'transport_2011-07-25_19-05-49_2.xml'), - os.path.join(self.this_directory, 'fixtures', - 'transportation', 'instances', - 'transport_2011-07-25_19-05-49_2', media_file)) - - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + self._make_submission_w_attachment( + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + "transport_2011-07-25_19-05-49_2", + "transport_2011-07-25_19-05-49_2.xml", + ), + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + "transport_2011-07-25_19-05-49_2", + media_file, + ), + ) + + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk - response = view(request, pk=formid, format='xml') + response = view(request, pk=formid, format="xml") self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) # Ensure response is renderable response.render() - self.assertEqual(response.accepted_media_type, 'application/xml') + self.assertEqual(response.accepted_media_type, "application/xml") instance = self.xform.instances.first() - returned_xml = response.content.decode('utf-8') - server_time = ET.fromstring(returned_xml).attrib.get('serverTime') + returned_xml = response.content.decode("utf-8") + server_time = ET.fromstring(returned_xml).attrib.get("serverTime") edited = instance.last_edited is not None submission_time = instance.date_created.strftime(MONGO_STRFTIME) attachment = instance.attachments.first() @@ -2750,19 +2748,19 @@ def test_data_list_xml_format(self): f' lastModified="{instance.date_modified.isoformat()}" mediaAllReceived="{instance.media_all_received}" mediaCount="{ instance.media_count }" objectID="{instance.id}" reviewComment="" reviewStatus=""' # noqa f' status="{instance.status}" submissionTime="{submission_time}" submittedBy="{instance.user.username}" totalMedia="{ instance.total_media }">' # noqa f'' # noqa - '' - 'none' # noqa - '' # noqa - '' + "" + "none" # noqa + "" # noqa + "" '1335783522563.jpg' - 'uuid:5b2cc313-fc09-437e-8149-fcd32f695d41' # noqa - '' - '' - '' - f'{attachment.id}1335783522563.jpg{instance.xform.id}{ attachment.media_file.name }{ instance.id }image/jpeg/api/v1/files/{attachment.id}?filename={ attachment.media_file.name }/api/v1/files/{attachment.id}?filename={ attachment.media_file.name }&suffix=small/api/v1/files/{attachment.id}?filename={ attachment.media_file.name }&suffix=medium' # noqa - '' - '' - '' + "uuid:5b2cc313-fc09-437e-8149-fcd32f695d41" # noqa + "" + "" + "" + f"{attachment.id}1335783522563.jpg{instance.xform.id}{ attachment.media_file.name }{ instance.id }image/jpeg/api/v1/files/{attachment.id}?filename={ attachment.media_file.name }/api/v1/files/{attachment.id}?filename={ attachment.media_file.name }&suffix=small/api/v1/files/{attachment.id}?filename={ attachment.media_file.name }&suffix=medium" # noqa + "" + "" + "" ) self.assertEqual(expected_xml, returned_xml) @@ -2770,27 +2768,44 @@ def test_invalid_xml_elements_not_present(self): """ Test invalid XML elements such as #text are not present in XML Response """ - self._make_submission(os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', 'transport_2011-07-25_19-05-49_2', - 'transport_2011-07-25_19-05-49_2.xml')) + self._make_submission( + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + "transport_2011-07-25_19-05-49_2", + "transport_2011-07-25_19-05-49_2.xml", + ) + ) media_file = "1335783522563.jpg" - self._make_submission_w_attachment(os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', 'transport_2011-07-25_19-05-49_2', - 'transport_2011-07-25_19-05-49_2.xml'), - os.path.join(self.this_directory, 'fixtures', - 'transportation', 'instances', - 'transport_2011-07-25_19-05-49_2', media_file)) + self._make_submission_w_attachment( + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + "transport_2011-07-25_19-05-49_2", + "transport_2011-07-25_19-05-49_2.xml", + ), + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + "transport_2011-07-25_19-05-49_2", + media_file, + ), + ) - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) - response = view(request, pk=self.xform.pk, format='xml') + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) + response = view(request, pk=self.xform.pk, format="xml") self.assertEqual(response.status_code, 200) # Ensure XML is well formed response.render() - returned_xml = response.content.decode('utf-8') + returned_xml = response.content.decode("utf-8") ET.fromstring(returned_xml) @override_settings(STREAM_DATA=True) @@ -2799,30 +2814,27 @@ def test_data_list_xml_format_no_data(self): # create form xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/forms/tutorial/tutorial.xlsx" + "../fixtures/forms/tutorial/tutorial.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk query_str = '{"_date_modified":{"$gt":"2020-01-22T11:42:20"}}' - request = self.factory.get('/?query=%s' % query_str, **self.extra) - response = view(request, pk=formid, format='xml') + request = self.factory.get("/?query=%s" % query_str, **self.extra) + response = view(request, pk=formid, format="xml") self.assertEqual(response.status_code, 200) - returned_xml = ( - ''.join([i.decode('utf-8') for i in response.streaming_content]) - - ) + returned_xml = "".join([i.decode("utf-8") for i in response.streaming_content]) - server_time = ET.fromstring(returned_xml).attrib.get('serverTime') + server_time = ET.fromstring(returned_xml).attrib.get("serverTime") expected_xml = ( '\n' f'' - '' + "" ) self.assertEqual(expected_xml, returned_xml) @@ -2833,30 +2845,30 @@ def test_data_list_xml_format_sort(self): responses """ self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk - expected_order = list(Instance.objects.filter( - xform=self.xform).order_by( - '-date_modified').values_list('id', flat=True)) - request = self.factory.get( - '/?sort=-_date_modified', **self.extra) - response = view(request, pk=formid, format='xml') + expected_order = list( + Instance.objects.filter(xform=self.xform) + .order_by("-date_modified") + .values_list("id", flat=True) + ) + request = self.factory.get("/?sort=-_date_modified", **self.extra) + response = view(request, pk=formid, format="xml") self.assertEqual(response.status_code, 200) - items_in_order = [ - int(data.get('@objectID')) for data in response.data] + items_in_order = [int(data.get("@objectID")) for data in response.data] self.assertEqual(expected_order, items_in_order) # Test `last_edited` field is sorted correctly - expected_order = list(Instance.objects.filter( - xform=self.xform).order_by( - 'last_edited').values_list('id', flat=True)) - request = self.factory.get( - '/?sort=_last_edited', **self.extra) - response = view(request, pk=formid, format='xml') + expected_order = list( + Instance.objects.filter(xform=self.xform) + .order_by("last_edited") + .values_list("id", flat=True) + ) + request = self.factory.get("/?sort=_last_edited", **self.extra) + response = view(request, pk=formid, format="xml") self.assertEqual(response.status_code, 200) - items_in_order = [ - int(data.get('@objectID')) for data in response.data] + items_in_order = [int(data.get("@objectID")) for data in response.data] self.assertEqual(expected_order, items_in_order) @override_settings(SUBMISSION_RETRIEVAL_THRESHOLD=1) @@ -2866,94 +2878,94 @@ def test_data_paginated_past_threshold(self): the requested data surpasses the SUBMISSION_RETRIEVAL_THRESHOLD """ self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk count = self.xform.instances.all().count() self.assertTrue(count > 1) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) - self.assertIn('Link', response) + self.assertIn("Link", response) self.assertEqual( - response['Link'], + response["Link"], '; rel="next", ' - '; rel="last"') + '; rel="last"', + ) class TestOSM(TestAbstractViewSet): """ Test OSM endpoints in data_viewset. """ + def setUp(self): super(TestOSM, self).setUp() self._login_user_and_profile() self.factory = RequestFactory() - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} def test_data_retrieve_instance_osm_format(self): filenames = [ - 'OSMWay234134797.osm', - 'OSMWay34298972.osm', + "OSMWay234134797.osm", + "OSMWay34298972.osm", ] - osm_fixtures_dir = os.path.realpath(os.path.join( - os.path.dirname(__file__), '..', 'fixtures', 'osm')) - paths = [ - os.path.join(osm_fixtures_dir, filename) - for filename in filenames] - xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') - combined_osm_path = os.path.join(osm_fixtures_dir, 'combined.osm') + osm_fixtures_dir = os.path.realpath( + os.path.join(os.path.dirname(__file__), "..", "fixtures", "osm") + ) + paths = [os.path.join(osm_fixtures_dir, filename) for filename in filenames] + xlsform_path = os.path.join(osm_fixtures_dir, "osm.xlsx") + combined_osm_path = os.path.join(osm_fixtures_dir, "combined.osm") self._publish_xls_form_to_project(xlsform_path=xlsform_path) - submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') - files = [open(path, 'rb') for path in paths] - count = Attachment.objects.filter(extension='osm').count() + submission_path = os.path.join(osm_fixtures_dir, "instance_a.xml") + files = [open(path, "rb") for path in paths] + count = Attachment.objects.filter(extension="osm").count() self._make_submission(submission_path, media_file=files) - self.assertTrue( - Attachment.objects.filter(extension='osm').count() > count) + self.assertTrue(Attachment.objects.filter(extension="osm").count() > count) formid = self.xform.pk - dataid = self.xform.instances.latest('date_created').pk - request = self.factory.get('/', **self.extra) + dataid = self.xform.instances.latest("date_created").pk + request = self.factory.get("/", **self.extra) # look at the data/[pk]/[dataid].osm endpoint - view = DataViewSet.as_view({'get': 'list'}) - response1 = view(request, pk=formid, format='osm') + view = DataViewSet.as_view({"get": "list"}) + response1 = view(request, pk=formid, format="osm") self.assertEqual(response1.status_code, 200) # look at the data/[pk]/[dataid].osm endpoint - view = DataViewSet.as_view({'get': 'retrieve'}) - response = view(request, pk=formid, dataid=dataid, format='osm') + view = DataViewSet.as_view({"get": "retrieve"}) + response = view(request, pk=formid, dataid=dataid, format="osm") self.assertEqual(response.status_code, 200) - with open(combined_osm_path, encoding='utf-8') as f: + with open(combined_osm_path, encoding="utf-8") as f: osm = f.read() response.render() - self.assertMultiLineEqual(response.content.decode('utf-8').strip(), - osm.strip()) + self.assertMultiLineEqual( + response.content.decode("utf-8").strip(), osm.strip() + ) # look at the data/[pk].osm endpoint - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='osm') + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="osm") self.assertEqual(response.status_code, 200) response.render() response1.render() self.assertMultiLineEqual( - response1.content.decode('utf-8').strip(), osm.strip()) + response1.content.decode("utf-8").strip(), osm.strip() + ) self.assertMultiLineEqual( - response.content.decode('utf-8').strip(), osm.strip()) + response.content.decode("utf-8").strip(), osm.strip() + ) # filter using value that exists request = self.factory.get( - '/', - data={"query": '{"osm_road": "OSMWay234134797.osm"}'}, - **self.extra) + "/", data={"query": '{"osm_road": "OSMWay234134797.osm"}'}, **self.extra + ) response = view(request, pk=formid) self.assertEqual(len(response.data), 1) # filter using value that doesn't exists request = self.factory.get( - '/', - data={"query": '{"osm_road": "OSMWay123456789.osm"}'}, - **self.extra) + "/", data={"query": '{"osm_road": "OSMWay123456789.osm"}'}, **self.extra + ) response = view(request, pk=formid) self.assertEqual(len(response.data), 0) diff --git a/onadata/apps/api/tests/viewsets/test_dataview_viewset.py b/onadata/apps/api/tests/viewsets/test_dataview_viewset.py index 087e39d4f5..84305423b3 100644 --- a/onadata/apps/api/tests/viewsets/test_dataview_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_dataview_viewset.py @@ -1,7 +1,8 @@ +# -*- coding: utf-8 -*- +"""Test DataViewViewSet""" import json import os import csv -from builtins import open from datetime import datetime, timedelta from django.conf import settings @@ -14,8 +15,7 @@ from onadata.libs.permissions import ReadOnlyRole from onadata.apps.logger.models.data_view import DataView -from onadata.apps.api.tests.viewsets.test_abstract_viewset import\ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.viewer.models.export import Export from onadata.apps.api.viewsets.project_viewset import ProjectViewSet from onadata.apps.api.viewsets.dataview_viewset import DataViewViewSet @@ -24,83 +24,100 @@ from onadata.libs.utils.cache_tools import ( DATAVIEW_COUNT, DATAVIEW_LAST_SUBMISSION_TIME, - PROJECT_LINKED_DATAVIEWS) + PROJECT_LINKED_DATAVIEWS, +) from onadata.libs.utils.common_tags import EDITED, MONGO_STRFTIME from onadata.apps.api.viewsets.xform_viewset import XFormViewSet from onadata.libs.utils.common_tools import ( - filename_from_disposition, get_response_content) + filename_from_disposition, + get_response_content, +) class TestDataViewViewSet(TestAbstractViewSet): - def setUp(self): - super(TestDataViewViewSet, self).setUp() + super().setUp() xlsform_path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", "fixtures", - "tutorial.xlsx") + settings.PROJECT_ROOT, "libs", "tests", "utils", "fixtures", "tutorial.xlsx" + ) self._publish_xls_form_to_project(xlsform_path=xlsform_path) - for x in range(1, 9): + for fixture in range(1, 9): path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - 'tutorial', 'instances', 'uuid{}'.format(x), 'submission.xml') + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial", + "instances", + f"uuid{fixture}", + "submission.xml", + ) self._make_submission(path) - x += 1 - self.view = DataViewViewSet.as_view({ - 'post': 'create', - 'put': 'update', - 'patch': 'partial_update', - 'delete': 'destroy', - 'get': 'retrieve' - }) + self.view = DataViewViewSet.as_view( + { + "post": "create", + "put": "update", + "patch": "partial_update", + "delete": "destroy", + "get": "retrieve", + } + ) def test_create_dataview(self): self._create_dataview() + # pylint: disable=invalid-name def test_dataview_with_attachment_field(self): - view = DataViewViewSet.as_view({ - 'get': 'data' - }) + view = DataViewViewSet.as_view({"get": "data"}) media_file = "test-image.png" attachment_file_path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - media_file) + settings.PROJECT_ROOT, "libs", "tests", "utils", "fixtures", media_file + ) submission_file_path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - 'tutorial', 'instances', 'uuid10', 'submission.xml') + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial", + "instances", + "uuid10", + "submission.xml", + ) # make a submission with an attachment - with open(attachment_file_path, 'rb') as f: + with open(attachment_file_path, "rb") as f: self._make_submission(submission_file_path, media_file=f) data = { - 'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, + "name": "My DataView", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", # ensure there's an attachment column(photo) in you dataview - 'columns': '["name", "age", "gender", "photo"]' + "columns": '["name", "age", "gender", "photo"]', } self._create_dataview(data=data) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) - for a in response.data: + for item in response.data: # retrieve the instance with attachment - if a.get('photo') == media_file: - instance_with_attachment = a + if item.get("photo") == media_file: + instance_with_attachment = item self.assertTrue(instance_with_attachment) - attachment_info = instance_with_attachment.get('_attachments')[0] - self.assertEquals(u'image/png', attachment_info.get(u'mimetype')) - self.assertEquals( - u'{}/attachments/{}_{}/{}'.format( - self.user.username, self.xform.id, self.xform.id_string, - media_file), - attachment_info.get(u'filename')) - self.assertEquals(response.status_code, 200) + attachment_info = instance_with_attachment.get("_attachments")[0] + self.assertEqual("image/png", attachment_info.get("mimetype")) + self.assertEqual( + f"{self.user.username}/attachments/{self.xform.id}_{self.xform.id_string}/{media_file}", + attachment_info.get("filename"), + ) + self.assertEqual(response.status_code, 200) + # pylint: disable=invalid-name def test_get_dataview_form_definition(self): self._create_dataview() @@ -111,29 +128,33 @@ def test_get_dataview_form_definition(self): "id_string": "tutorial", "type": "survey", } - self.view = DataViewViewSet.as_view({ - 'get': 'form', - }) - request = self.factory.get('/', **self.extra) + self.view = DataViewViewSet.as_view( + { + "get": "form", + } + ) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) # JSON format - response = self.view(request, pk=self.data_view.pk, format='json') + response = self.view(request, pk=self.data_view.pk, format="json") self.assertEqual(response.status_code, 200) self.assertDictContainsSubset(data, response.data) def test_get_dataview_form_details(self): self._create_dataview() - self.view = DataViewViewSet.as_view({ - 'get': 'form_details', - }) - request = self.factory.get('/', **self.extra) + self.view = DataViewViewSet.as_view( + { + "get": "form_details", + } + ) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) - response = self.view(request, pk=self.data_view.pk, format='json') + response = self.view(request, pk=self.data_view.pk, format="json") self.assertEqual(response.status_code, 200) self.assertIn("title", response.data) @@ -144,80 +165,84 @@ def test_get_dataview_form_details(self): def test_get_dataview(self): self._create_dataview() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['dataviewid'], self.data_view.pk) - self.assertEquals(response.data['name'], 'My DataView') - self.assertEquals(response.data['xform'], - 'http://testserver/api/v1/forms/%s' % self.xform.pk) - self.assertEquals(response.data['project'], - 'http://testserver/api/v1/projects/%s' - % self.project.pk) - self.assertEquals(response.data['columns'], - ["name", "age", "gender"]) - self.assertEquals(response.data['query'], - [{"column": "age", "filter": ">", "value": "20"}, - {"column": "age", "filter": "<", "value": "50"}]) - self.assertEquals(response.data['url'], - 'http://testserver/api/v1/dataviews/%s' - % self.data_view.pk) - self.assertEquals(response.data['last_submission_time'], - '2015-03-09T13:34:05') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["dataviewid"], self.data_view.pk) + self.assertEqual(response.data["name"], "My DataView") + self.assertEqual( + response.data["xform"], f"http://testserver/api/v1/forms/{self.xform.pk}" + ) + self.assertEqual( + response.data["project"], + f"http://testserver/api/v1/projects/{self.project.pk}", + ) + self.assertEqual(response.data["columns"], ["name", "age", "gender"]) + self.assertEqual( + response.data["query"], + [ + {"column": "age", "filter": ">", "value": "20"}, + {"column": "age", "filter": "<", "value": "50"}, + ], + ) + self.assertEqual( + response.data["url"], + f"http://testserver/api/v1/dataviews/{self.data_view.pk}", + ) + self.assertEqual(response.data["last_submission_time"], "2015-03-09T13:34:05") # Public self.project.shared = True self.project.save() - anon_request = self.factory.get('/') + anon_request = self.factory.get("/") anon_response = self.view(anon_request, pk=self.data_view.pk) - self.assertEquals(anon_response.status_code, 200) + self.assertEqual(anon_response.status_code, 200) # Private self.project.shared = False self.project.save() - anon_request = self.factory.get('/') + anon_request = self.factory.get("/") anon_response = self.view(anon_request, pk=self.data_view.pk) - self.assertEquals(anon_response.status_code, 404) + self.assertEqual(anon_response.status_code, 404) def test_update_dataview(self): self._create_dataview() data = { - 'name': "My DataView updated", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[{"column":"age","filter":">","value":"20"}]' + "name": "My DataView updated", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender"]', + "query": '[{"column":"age","filter":">","value":"20"}]', } - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['name'], 'My DataView updated') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["name"], "My DataView updated") - self.assertEquals(response.data['columns'], - ["name", "age", "gender"]) + self.assertEqual(response.data["columns"], ["name", "age", "gender"]) - self.assertEquals(response.data['query'], - [{"column": "age", "filter": ">", "value": "20"}]) + self.assertEqual( + response.data["query"], [{"column": "age", "filter": ">", "value": "20"}] + ) def test_patch_dataview(self): self._create_dataview() data = { - 'name': "My DataView updated", + "name": "My DataView updated", } - request = self.factory.patch('/', data=data, **self.extra) + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['name'], 'My DataView updated') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["name"], "My DataView updated") def test_soft_delete_dataview(self): """ @@ -228,13 +253,13 @@ def test_soft_delete_dataview(self): self.assertIsNone(self.data_view.deleted_at) self.assertNotIn("-deleted-at-", self.data_view.name) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=dataview_id) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['name'], u'My DataView') + self.assertEqual(response.data["name"], "My DataView") # delete - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = self.view(request, pk=dataview_id) self.assertEqual(response.status_code, 204) @@ -242,118 +267,121 @@ def test_soft_delete_dataview(self): data_view = DataView.objects.get(pk=dataview_id) self.assertIsNotNone(data_view.deleted_at) self.assertIn("-deleted-at-", data_view.name) - self.assertEqual(data_view.deleted_by.username, u'bob') + self.assertEqual(data_view.deleted_by.username, "bob") + # pylint: disable=invalid-name def test_soft_deleted_dataview_not_in_forms_list(self): self._create_dataview() - get_form_request = self.factory.get('/', **self.extra) + get_form_request = self.factory.get("/", **self.extra) xform_serializer = XFormSerializer( - self.xform, - context={'request': get_form_request}) + self.xform, context={"request": get_form_request} + ) - self.assertIsNotNone(xform_serializer.data['data_views']) + self.assertIsNotNone(xform_serializer.data["data_views"]) - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 204) + self.assertEqual(response.status_code, 204) xform_serializer = XFormSerializer( - self.xform, - context={'request': get_form_request}) - self.assertEquals(xform_serializer.data['data_views'], []) + self.xform, context={"request": get_form_request} + ) + self.assertEqual(xform_serializer.data["data_views"], []) + # pylint: disable=invalid-name def test_soft_deleted_dataview_not_in_project(self): """ Test that once a filtered dataset is soft deleted it does not appear in the list of forms for a project """ self._create_dataview() - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"get": "retrieve"}) # assert that dataview is in the returned list - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertIsNotNone(response.data['data_views']) + self.assertIsNotNone(response.data["data_views"]) # delete dataview - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 204) + self.assertEqual(response.status_code, 204) # assert that deleted dataview is not in the returned list - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertEqual(response.data['data_views'], []) + self.assertEqual(response.data["data_views"], []) def test_list_dataview(self): self._create_dataview() data = { - 'name': "My DataView2", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[{"column":"age","filter":">","value":"20"}]' + "name": "My DataView2", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender"]', + "query": '[{"column":"age","filter":">","value":"20"}]', } self._create_dataview(data=data) - view = DataViewViewSet.as_view({ - 'get': 'list', - }) + view = DataViewViewSet.as_view( + { + "get": "list", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 2) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 2) - anon_request = request = self.factory.get('/') + anon_request = request = self.factory.get("/") anon_response = view(anon_request) - self.assertEquals(anon_response.status_code, 401) + self.assertEqual(anon_response.status_code, 401) def test_get_dataview_no_perms(self): self._create_dataview() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 404) + self.assertEqual(response.status_code, 404) # assign alice the perms ReadOnlyRole.add(self.user, self.data_view.project) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) + # pylint: disable=invalid-name def test_dataview_data_filter_integer(self): data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[{"column":"age","filter":">","value":"20"},' - '{"column":"age","filter":"<","value":"50"}]' + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender"]', + "query": '[{"column":"age","filter":">","value":"20"},' + '{"column":"age","filter":"<","value":"50"}]', } self._create_dataview(data=data) - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 3) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 3) self.assertIn("_id", response.data[0]) def test_dataview_data_filter_decimal(self): @@ -362,202 +390,228 @@ def test_dataview_data_filter_decimal(self): """ # publish form with decimal field and make submissions path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", "fixtures", - "age_decimal", "age_decimal.xlsx") + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "age_decimal", + "age_decimal.xlsx", + ) self._publish_xls_form_to_project(xlsform_path=path) - for x in range(1, 3): + for fixture in range(1, 3): path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - 'age_decimal', 'instances', 'submission{}.xml'.format(x), ) + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "age_decimal", + "instances", + f"submission{fixture}.xml", + ) self._make_submission(path) - x += 1 # create a dataview using filter age > 30 data = { - 'name': "My Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age"]', - 'query': '[{"column":"age","filter":">","value":"30"}]' + "name": "My Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age"]', + "query": '[{"column":"age","filter":">","value":"30"}]', } self._create_dataview(data=data) - view = DataViewViewSet.as_view({ - 'get': 'data', - }) - request = self.factory.get('/', **self.extra) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 1) - self.assertEquals(response.data[0]['age'], 31) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 1) + self.assertEqual(response.data[0]["age"], 31) def test_dataview_data_filter_date(self): data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "gender", "_submission_time"]', - 'query': '[{"column":"_submission_time",' - '"filter":">=","value":"2015-01-01T00:00:00"}]' + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "gender", "_submission_time"]', + "query": '[{"column":"_submission_time",' + '"filter":">=","value":"2015-01-01T00:00:00"}]', } self._create_dataview(data=data) - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 7) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 7) self.assertIn("_id", response.data[0]) + # pylint: disable=invalid-name def test_dataview_data_filter_string(self): data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "gender", "_submission_time"]', - 'query': '[{"column":"gender","filter":"<>","value":"male"}]' + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "gender", "_submission_time"]', + "query": '[{"column":"gender","filter":"<>","value":"male"}]', } self._create_dataview(data=data) - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 1) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 1) + # pylint: disable=invalid-name def test_dataview_data_filter_condition(self): data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "gender", "age"]', - 'query': '[{"column":"name","filter":"=","value":"Fred",' - ' "condition":"or"},' - '{"column":"name","filter":"=","value":"Kameli",' - ' "condition":"or"},' - '{"column":"gender","filter":"=","value":"male"}]' + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "gender", "age"]', + "query": '[{"column":"name","filter":"=","value":"Fred",' + ' "condition":"or"},' + '{"column":"name","filter":"=","value":"Kameli",' + ' "condition":"or"},' + '{"column":"gender","filter":"=","value":"male"}]', } self._create_dataview(data=data) - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 2) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 2) self.assertIn("_id", response.data[0]) def test_dataview_invalid_filter(self): data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "gender", "age"]', - 'query': '[{"column":"name","filter":"<=>","value":"Fred",' - ' "condition":"or"}]' + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "gender", "age"]', + "query": '[{"column":"name","filter":"<=>","value":"Fred",' + ' "condition":"or"}]', } - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(response.data, - {'query': [u'Filter not supported']}) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.data, {"query": ["Filter not supported"]}) def test_dataview_sql_injection(self): data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "gender", "age"]', - 'query': '[{"column":"age","filter":"=",' - '"value":"1;UNION ALL SELECT NULL,version()' - ',NULL LIMIT 1 OFFSET 1--;"}]' + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "gender", "age"]', + "query": '[{"column":"age","filter":"=",' + '"value":"1;UNION ALL SELECT NULL,version()' + ',NULL LIMIT 1 OFFSET 1--;"}]', } - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = self.view(request) + self.assertEqual(response.status_code, 400) + self.assertIn("detail", response.data) - self.assertEquals(response.status_code, 400) - self.assertIn('detail', response.data) - - self.assertTrue(str(response.data.get('detail')) - .startswith("invalid input syntax for type integer")) + self.assertTrue( + str(response.data.get("detail")).startswith( + "invalid input syntax for type integer" + ), + response.data, + ) def test_dataview_invalid_columns(self): data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': 'age' + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": "age", } - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertIn(response.data['columns'][0], - ['Expecting value: line 1 column 1 (char 0)', - 'No JSON object could be decoded']) + self.assertEqual(response.status_code, 400) + self.assertIn( + response.data["columns"][0], + [ + "Expecting value: line 1 column 1 (char 0)", + "No JSON object could be decoded", + ], + ) def test_dataview_invalid_query(self): data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["age"]', - 'query': 'age=10' + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["age"]', + "query": "age=10", } - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertIn(response.data['query'][0], - ['Expecting value: line 1 column 1 (char 0)', - u'No JSON object could be decoded']) + self.assertEqual(response.status_code, 400) + self.assertIn( + response.data["query"][0], + [ + "Expecting value: line 1 column 1 (char 0)", + "No JSON object could be decoded", + ], + ) + # pylint: disable=invalid-name def test_dataview_query_not_required(self): data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["age"]', + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["age"]', } self._create_dataview(data=data) - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 8) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 8) self.assertIn("_id", response.data[0]) self.assertIn(EDITED, response.data[0]) @@ -566,209 +620,226 @@ def test_csv_export_dataview(self): self._create_dataview() count = Export.objects.all().count() - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) - response = view(request, pk=self.data_view.pk, format='csv') + request = self.factory.get("/", **self.extra) + response = view(request, pk=self.data_view.pk, format="csv") self.assertEqual(response.status_code, 200) - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], 'application/csv') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/csv") + content_disposition = headers["Content-Disposition"] filename = filename_from_disposition(content_disposition) - basename, ext = os.path.splitext(filename) - self.assertEqual(ext, '.csv') + _basename, ext = os.path.splitext(filename) + self.assertEqual(ext, ".csv") content = get_response_content(response) - test_file_path = os.path.join(settings.PROJECT_ROOT, 'apps', - 'viewer', 'tests', 'fixtures', - 'dataview.csv') - with open(test_file_path, encoding='utf-8') as test_file: + test_file_path = os.path.join( + settings.PROJECT_ROOT, "apps", "viewer", "tests", "fixtures", "dataview.csv" + ) + with open(test_file_path, encoding="utf-8") as test_file: self.assertEqual(content, test_file.read()) def test_csvzip_export_dataview(self): self._create_dataview() count = Export.objects.all().count() - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) - response = view(request, pk=self.data_view.pk, format='csvzip') + request = self.factory.get("/", **self.extra) + response = view(request, pk=self.data_view.pk, format="csvzip") self.assertEqual(response.status_code, 200) - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) - request = self.factory.get('/', **self.extra) - response = view(request, pk='[invalid pk]', format='csvzip') + request = self.factory.get("/", **self.extra) + response = view(request, pk="[invalid pk]", format="csvzip") self.assertEqual(response.status_code, 404) def test_zip_export_dataview(self): media_file = "test-image.png" attachment_file_path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - media_file) + settings.PROJECT_ROOT, "libs", "tests", "utils", "fixtures", media_file + ) submission_file_path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - 'tutorial', 'instances', 'uuid10', 'submission.xml') + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial", + "instances", + "uuid10", + "submission.xml", + ) # make a submission with an attachment - with open(attachment_file_path, 'rb') as f: + with open(attachment_file_path, "rb") as f: self._make_submission(submission_file_path, media_file=f) data = { - 'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "photo"]', - 'query': '[{"column":"age","filter":"=","value":"90"}]' + "name": "My DataView", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "photo"]', + "query": '[{"column":"age","filter":"=","value":"90"}]', } self._create_dataview(data) count = Export.objects.all().count() - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) self.assertEqual(1, len(response.data)) - request = self.factory.get('/', **self.extra) - response = view(request, pk=self.data_view.pk, format='zip') + request = self.factory.get("/", **self.extra) + response = view(request, pk=self.data_view.pk, format="zip") self.assertEqual(response.status_code, 200) - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], 'application/zip') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/zip") + content_disposition = headers["Content-Disposition"] filename = filename_from_disposition(content_disposition) - basename, ext = os.path.splitext(filename) - self.assertEqual(ext, '.zip') + _basename, ext = os.path.splitext(filename) + self.assertEqual(ext, ".zip") + # pylint: disable=invalid-name @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch('onadata.apps.api.viewsets.dataview_viewset.AsyncResult') + @patch("onadata.apps.api.viewsets.dataview_viewset.AsyncResult") def test_export_csv_dataview_data_async(self, async_result): self._create_dataview() self._publish_xls_form_to_project() - view = DataViewViewSet.as_view({ - 'get': 'export_async', - }) + view = DataViewViewSet.as_view( + { + "get": "export_async", + } + ) - request = self.factory.get('/', data={"format": "csv"}, - **self.extra) + request = self.factory.get("/", data={"format": "csv"}, **self.extra) response = view(request, pk=self.data_view.pk) self.assertIsNotNone(response.data) self.assertEqual(response.status_code, 202) - self.assertTrue('job_uuid' in response.data) - task_id = response.data.get('job_uuid') + self.assertTrue("job_uuid" in response.data) + task_id = response.data.get("job_uuid") - export_pk = Export.objects.all().order_by('pk').reverse()[0].pk + export_pk = Export.objects.all().order_by("pk").reverse()[0].pk # metaclass for mocking results - job = type('AsyncResultMock', (), - {'state': 'SUCCESS', 'result': export_pk}) + job = type("AsyncResultMock", (), {"state": "SUCCESS", "result": export_pk}) async_result.return_value = job - get_data = {'job_uuid': task_id} - request = self.factory.get('/', data=get_data, **self.extra) + get_data = {"job_uuid": task_id} + request = self.factory.get("/", data=get_data, **self.extra) response = view(request, pk=self.data_view.pk) - self.assertIn('export_url', response.data) + self.assertIn("export_url", response.data) self.assertTrue(async_result.called) self.assertEqual(response.status_code, 202) export = Export.objects.get(task_id=task_id) self.assertTrue(export.is_successful) + # pylint: disable=invalid-name @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch('onadata.apps.api.viewsets.dataview_viewset.AsyncResult') + @patch("onadata.apps.api.viewsets.dataview_viewset.AsyncResult") def test_export_csv_dataview_with_labels_async(self, async_result): self._create_dataview() self._publish_xls_form_to_project() - view = DataViewViewSet.as_view({ - 'get': 'export_async', - }) + view = DataViewViewSet.as_view( + { + "get": "export_async", + } + ) - request = self.factory.get('/', data={"format": "csv", - 'include_labels': 'true'}, - **self.extra) + request = self.factory.get( + "/", data={"format": "csv", "include_labels": "true"}, **self.extra + ) response = view(request, pk=self.data_view.pk) self.assertIsNotNone(response.data) self.assertEqual(response.status_code, 202) - self.assertTrue('job_uuid' in response.data) - task_id = response.data.get('job_uuid') + self.assertTrue("job_uuid" in response.data) + task_id = response.data.get("job_uuid") - export_pk = Export.objects.all().order_by('pk').reverse()[0].pk + export_pk = Export.objects.all().order_by("pk").reverse()[0].pk # metaclass for mocking results - job = type('AsyncResultMock', (), - {'state': 'SUCCESS', 'result': export_pk}) + job = type("AsyncResultMock", (), {"state": "SUCCESS", "result": export_pk}) async_result.return_value = job - get_data = {'job_uuid': task_id} - request = self.factory.get('/', data=get_data, **self.extra) + get_data = {"job_uuid": task_id} + request = self.factory.get("/", data=get_data, **self.extra) response = view(request, pk=self.data_view.pk) - self.assertIn('export_url', response.data) + self.assertIn("export_url", response.data) self.assertTrue(async_result.called) self.assertEqual(response.status_code, 202) export = Export.objects.get(task_id=task_id) self.assertTrue(export.is_successful) - with default_storage.open(export.filepath, 'r') as f: + with default_storage.open(export.filepath, "r") as f: csv_reader = csv.reader(f) next(csv_reader) labels = next(csv_reader) - self.assertIn( - 'Gender', labels - ) + self.assertIn("Gender", labels) + # pylint: disable=invalid-name @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch('onadata.apps.api.viewsets.dataview_viewset.AsyncResult') + @patch("onadata.apps.api.viewsets.dataview_viewset.AsyncResult") def test_export_xls_dataview_with_labels_async(self, async_result): self._create_dataview() self._publish_xls_form_to_project() - view = DataViewViewSet.as_view({ - 'get': 'export_async', - }) + view = DataViewViewSet.as_view( + { + "get": "export_async", + } + ) - request = self.factory.get('/', data={"format": "xls", - "force_xlsx": 'true', - 'include_labels': 'true'}, - **self.extra) + request = self.factory.get( + "/", + data={"format": "xls", "force_xlsx": "true", "include_labels": "true"}, + **self.extra, + ) response = view(request, pk=self.data_view.pk) self.assertIsNotNone(response.data) self.assertEqual(response.status_code, 202) - self.assertTrue('job_uuid' in response.data) - task_id = response.data.get('job_uuid') + self.assertTrue("job_uuid" in response.data) + task_id = response.data.get("job_uuid") - export_pk = Export.objects.all().order_by('pk').reverse()[0].pk + export_pk = Export.objects.all().order_by("pk").reverse()[0].pk # metaclass for mocking results - job = type('AsyncResultMock', (), - {'state': 'SUCCESS', 'result': export_pk}) + job = type("AsyncResultMock", (), {"state": "SUCCESS", "result": export_pk}) async_result.return_value = job - get_data = {'job_uuid': task_id} - request = self.factory.get('/', data=get_data, **self.extra) + get_data = {"job_uuid": task_id} + request = self.factory.get("/", data=get_data, **self.extra) response = view(request, pk=self.data_view.pk) - self.assertIn('export_url', response.data) + self.assertIn("export_url", response.data) self.assertTrue(async_result.called) self.assertEqual(response.status_code, 202) @@ -777,201 +848,206 @@ def test_export_xls_dataview_with_labels_async(self, async_result): workbook = load_workbook(export.full_filepath) sheet_name = workbook.get_sheet_names()[0] main_sheet = workbook.get_sheet_by_name(sheet_name) - self.assertIn('Gender', tuple(main_sheet.values)[1]) + self.assertIn("Gender", tuple(main_sheet.values)[1]) self.assertEqual(len(tuple(main_sheet.values)), 5) - def _test_csv_export_with_hxl_support(self, name, columns, expected_output): # noqa + def _test_csv_export_with_hxl_support(self, name, columns, expected_output): # noqa data = { - 'name': name, - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': columns, - 'query': '[]' + "name": name, + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": columns, + "query": "[]", } self._create_dataview(data=data) dataview_pk = DataView.objects.last().pk - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) request = self.factory.get( - '/', data={"format": "csv", "include_hxl": True}, **self.extra) + "/", data={"format": "csv", "include_hxl": True}, **self.extra + ) response = view(request, pk=dataview_pk) - self.assertIsNotNone( - next(response.streaming_content), - expected_output - ) + self.assertIsNotNone(next(response.streaming_content), expected_output) + # pylint: disable=invalid-name def test_csv_export_with_hxl_support(self): self._publish_form_with_hxl_support() self._test_csv_export_with_hxl_support( - 'test name 1', - '["name"]', - 'name\nCristiano Ronaldo 1\nLionel Messi\n' + "test name 1", '["name"]', "name\nCristiano Ronaldo 1\nLionel Messi\n" ) self._test_csv_export_with_hxl_support( - 'test name 2', - '["age"]', - 'age\n#age,\n31\n29\n' + "test name 2", '["age"]', "age\n#age,\n31\n29\n" ) self._test_csv_export_with_hxl_support( - 'test name 3', + "test name 3", '["age", "name"]', - 'age,name\n#age,\n31,Cristiano Ronaldo\n29,Lionel Messi\n' + "age,name\n#age,\n31,Cristiano Ronaldo\n29,Lionel Messi\n", ) def test_get_charts_data(self): self._create_dataview() - self.view = DataViewViewSet.as_view({ - 'get': 'charts', - }) + self.view = DataViewViewSet.as_view( + { + "get": "charts", + } + ) data_view_data = DataView.query_data(self.data_view) - request = self.factory.get('/charts', **self.extra) + request = self.factory.get("/charts", **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - data = {'field_name': 'age'} - request = self.factory.get('/charts', data, **self.extra) + data = {"field_name": "age"} + request = self.factory.get("/charts", data, **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) - self.assertEqual(response.data['field_type'], 'integer') - self.assertEqual(response.data['field_name'], 'age') - self.assertEqual(response.data['data_type'], 'numeric') - self.assertEqual(len(response.data['data']), len(data_view_data)) + self.assertNotEqual(response.get("Cache-Control"), None) + self.assertEqual(response.data["field_type"], "integer") + self.assertEqual(response.data["field_name"], "age") + self.assertEqual(response.data["data_type"], "numeric") + self.assertEqual(len(response.data["data"]), len(data_view_data)) - data = {'field_xpath': 'age'} - request = self.factory.get('/charts', data, **self.extra) + data = {"field_xpath": "age"} + request = self.factory.get("/charts", data, **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) - self.assertEqual(response.data['field_type'], 'integer') - self.assertEqual(response.data['field_name'], 'age') - self.assertEqual(response.data['data_type'], 'numeric') - self.assertEqual(len(response.data['data']), len(data_view_data)) + self.assertNotEqual(response.get("Cache-Control"), None) + self.assertEqual(response.data["field_type"], "integer") + self.assertEqual(response.data["field_name"], "age") + self.assertEqual(response.data["data_type"], "numeric") + self.assertEqual(len(response.data["data"]), len(data_view_data)) + # pylint: disable=invalid-name def test_get_charts_data_for_submission_time_field(self): self._create_dataview() - self.view = DataViewViewSet.as_view({ - 'get': 'charts', - }) + self.view = DataViewViewSet.as_view( + { + "get": "charts", + } + ) - data = {'field_name': '_submission_time'} - request = self.factory.get('/charts', data, **self.extra) + data = {"field_name": "_submission_time"} + request = self.factory.get("/charts", data, **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) - self.assertEqual(response.data['field_type'], 'datetime') - self.assertEqual(response.data['field_name'], '_submission_time') - self.assertEqual(response.data['data_type'], 'time_based') + self.assertNotEqual(response.get("Cache-Control"), None) + self.assertEqual(response.data["field_type"], "datetime") + self.assertEqual(response.data["field_name"], "_submission_time") + self.assertEqual(response.data["data_type"], "time_based") - data = {'field_name': '_submitted_by'} - request = self.factory.get('/charts', data, **self.extra) + data = {"field_name": "_submitted_by"} + request = self.factory.get("/charts", data, **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) - self.assertEqual(response.data['field_type'], 'text') - self.assertEqual(response.data['field_name'], '_submitted_by') - self.assertEqual(response.data['data_type'], 'categorized') + self.assertNotEqual(response.get("Cache-Control"), None) + self.assertEqual(response.data["field_type"], "text") + self.assertEqual(response.data["field_name"], "_submitted_by") + self.assertEqual(response.data["data_type"], "categorized") - data = {'field_name': '_duration'} - request = self.factory.get('/charts', data, **self.extra) + data = {"field_name": "_duration"} + request = self.factory.get("/charts", data, **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) - self.assertEqual(response.data['field_type'], 'integer') - self.assertEqual(response.data['field_name'], '_duration') - self.assertEqual(response.data['data_type'], 'numeric') + self.assertNotEqual(response.get("Cache-Control"), None) + self.assertEqual(response.data["field_type"], "integer") + self.assertEqual(response.data["field_name"], "_duration") + self.assertEqual(response.data["data_type"], "numeric") def test_get_charts_data_for_grouped_field(self): data = { - 'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender", "a_group/grouped"]', - 'query': '[{"column":"age","filter":">","value":"20"}]' + "name": "My DataView", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender", "a_group/grouped"]', + "query": '[{"column":"age","filter":">","value":"20"}]', } self._create_dataview(data) - self.view = DataViewViewSet.as_view({ - 'get': 'charts', - }) + self.view = DataViewViewSet.as_view( + { + "get": "charts", + } + ) - request = self.factory.get('/charts', **self.extra) + request = self.factory.get("/charts", **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - data = {'field_name': 'grouped'} - request = self.factory.get('/charts', data, **self.extra) + data = {"field_name": "grouped"} + request = self.factory.get("/charts", data, **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) - self.assertEqual(response.data['field_type'], 'select one') - self.assertEqual(response.data['field_name'], 'grouped') - self.assertEqual(response.data['data_type'], 'categorized') - self.assertEqual(len(response.data['data']), 2) + self.assertNotEqual(response.get("Cache-Control"), None) + self.assertEqual(response.data["field_type"], "select one") + self.assertEqual(response.data["field_name"], "grouped") + self.assertEqual(response.data["data_type"], "categorized") + self.assertEqual(len(response.data["data"]), 2) + # pylint: disable=invalid-name def test_get_charts_data_field_not_in_dataview_columns(self): self._create_dataview() - self.view = DataViewViewSet.as_view({ - 'get': 'charts', - }) + self.view = DataViewViewSet.as_view( + { + "get": "charts", + } + ) - data = {'field_name': 'grouped'} - request = self.factory.get('/charts', data, **self.extra) + data = {"field_name": "grouped"} + request = self.factory.get("/charts", data, **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 404) def test_get_charts_data_with_empty_query(self): data = { - 'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[]' + "name": "My DataView", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender"]', + "query": "[]", } self._create_dataview(data) - self.view = DataViewViewSet.as_view({ - 'get': 'charts', - }) + self.view = DataViewViewSet.as_view( + { + "get": "charts", + } + ) data_view_data = DataView.query_data(self.data_view) - request = self.factory.get('/charts', **self.extra) + request = self.factory.get("/charts", **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - data = {'field_name': 'age'} - request = self.factory.get('/charts', data, **self.extra) + data = {"field_name": "age"} + request = self.factory.get("/charts", data, **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) - self.assertEqual(response.data['field_type'], 'integer') - self.assertEqual(response.data['field_name'], 'age') - self.assertEqual(response.data['data_type'], 'numeric') - self.assertEqual(len(response.data['data']), len(data_view_data)) + self.assertNotEqual(response.get("Cache-Control"), None) + self.assertEqual(response.data["field_type"], "integer") + self.assertEqual(response.data["field_name"], "age") + self.assertEqual(response.data["data_type"], "numeric") + self.assertEqual(len(response.data["data"]), len(data_view_data)) def test_geopoint_dataview(self): # Dataview with geolocation column selected. # -> instances_with_geopoints= True data = { - 'name': "My DataView1", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender", "location"]', - 'query': '[{"column":"age","filter":">","value":"20"}]' + "name": "My DataView1", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender", "location"]', + "query": '[{"column":"age","filter":">","value":"20"}]', } self._create_dataview(data) @@ -980,88 +1056,101 @@ def test_geopoint_dataview(self): # Dataview with geolocation column NOT selected # -> instances_with_geopoints= False data = { - 'name': "My DataView2", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[{"column":"age","filter":">","value":"20"}]' + "name": "My DataView2", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender"]', + "query": '[{"column":"age","filter":">","value":"20"}]', } self._create_dataview(data) self.assertFalse(self.data_view.instances_with_geopoints) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['dataviewid'], self.data_view.pk) - self.assertEquals(response.data['name'], 'My DataView2') - self.assertEquals(response.data['instances_with_geopoints'], False) - - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["dataviewid"], self.data_view.pk) + self.assertEqual(response.data["name"], "My DataView2") + self.assertEqual(response.data["instances_with_geopoints"], False) + + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) self.assertNotIn("location", response.data[0]) self.assertNotIn("_geolocation", response.data[0]) + # pylint: disable=invalid-name def test_geopoint_submission_dataview(self): data = { - 'name': "My DataView3", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender", "location"]', - 'query': '[{"column":"age","filter":">=","value":"87"}]' + "name": "My DataView3", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender", "location"]', + "query": '[{"column":"age","filter":">=","value":"87"}]', } self._create_dataview(data) self.assertTrue(self.data_view.instances_with_geopoints) # make submission with geopoint - path = os.path.join(settings.PROJECT_ROOT, 'libs', 'tests', "utils", - 'fixtures', 'tutorial', 'instances', - 'uuid{}'.format(9), 'submission.xml') + path = os.path.join( + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial", + "instances", + "uuid9", + "submission.xml", + ) self._make_submission(path) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['dataviewid'], self.data_view.pk) - self.assertEquals(response.data['name'], 'My DataView3') - self.assertEquals(response.data['instances_with_geopoints'], True) - - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["dataviewid"], self.data_view.pk) + self.assertEqual(response.data["name"], "My DataView3") + self.assertEqual(response.data["instances_with_geopoints"], True) + + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) self.assertIn("location", response.data[0]) self.assertIn("_geolocation", response.data[0]) + # pylint: disable=invalid-name def test_dataview_project_cache_cleared(self): self._create_dataview() - view = ProjectViewSet.as_view({ - 'get': 'retrieve', - }) + view = ProjectViewSet.as_view( + { + "get": "retrieve", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) - cached_dataviews = cache.get('{}{}'.format(PROJECT_LINKED_DATAVIEWS, - self.project.pk)) + cached_dataviews = cache.get(f"{PROJECT_LINKED_DATAVIEWS}{self.project.pk}") self.assertIsNotNone(cached_dataviews) @@ -1069,344 +1158,376 @@ def test_dataview_project_cache_cleared(self): self.data_view.name = "updated name" self.data_view.save() - updated_cache = cache.get('{}{}'.format(PROJECT_LINKED_DATAVIEWS, - self.project.pk)) + updated_cache = cache.get(f"{PROJECT_LINKED_DATAVIEWS}{self.project.pk}") self.assertIsNone(updated_cache) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) - cached_dataviews = cache.get('{}{}'.format(PROJECT_LINKED_DATAVIEWS, - self.project.pk)) + cached_dataviews = cache.get(f"{PROJECT_LINKED_DATAVIEWS}{self.project.pk}") self.assertIsNotNone(cached_dataviews) self.data_view.delete() - updated_cache = cache.get('{}{}'.format(PROJECT_LINKED_DATAVIEWS, - self.project.pk)) + updated_cache = cache.get(f"{PROJECT_LINKED_DATAVIEWS}{self.project.pk}") self.assertIsNone(updated_cache) + # pylint: disable=invalid-name def test_dataview_update_refreshes_cached_data(self): self._create_dataview() - cache.set('{}{}'.format(DATAVIEW_COUNT, self.data_view.xform.pk), 5) - cache.set('{}{}'.format(DATAVIEW_LAST_SUBMISSION_TIME, - self.data_view.xform.pk), - '2015-03-09T13:34:05') + cache.set(f"{DATAVIEW_COUNT}{self.data_view.xform.pk}", 5) + cache.set( + f"{DATAVIEW_LAST_SUBMISSION_TIME}{self.data_view.xform.pk}", + "2015-03-09T13:34:05", + ) self.data_view.name = "Updated Dataview" self.data_view.save() + self.assertIsNone(cache.get(f"{DATAVIEW_COUNT}{self.data_view.xform.pk}")) self.assertIsNone( - cache.get('{}{}'.format(DATAVIEW_COUNT, self.data_view.xform.pk))) - self.assertIsNone(cache.get('{}{}'.format( - DATAVIEW_LAST_SUBMISSION_TIME, self.data_view.xform.pk))) + cache.get(f"{DATAVIEW_LAST_SUBMISSION_TIME}{self.data_view.xform.pk}") + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.data_view.pk) expected_count = 3 expected_last_submission_time = "2015-03-09T13:34:05" - self.assertEquals(response.data['count'], expected_count) - self.assertEquals(response.data['last_submission_time'], - '2015-03-09T13:34:05') + self.assertEqual(response.data["count"], expected_count) + self.assertEqual(response.data["last_submission_time"], "2015-03-09T13:34:05") - cache_dict = cache.get('{}{}'.format(DATAVIEW_COUNT, - self.data_view.xform.pk)) - self.assertEquals(cache_dict.get(self.data_view.pk), expected_count) - self.assertEquals(cache.get('{}{}'.format( - DATAVIEW_LAST_SUBMISSION_TIME, self.data_view.xform.pk)), - expected_last_submission_time) + cache_dict = cache.get(f"{DATAVIEW_COUNT}{self.data_view.xform.pk}") + self.assertEqual(cache_dict.get(self.data_view.pk), expected_count) + self.assertEqual( + cache.get(f"{DATAVIEW_LAST_SUBMISSION_TIME}{self.data_view.xform.pk}"), + expected_last_submission_time, + ) + # pylint: disable=invalid-name def test_export_dataview_not_affected_by_normal_exports(self): count = Export.objects.all().count() - view = XFormViewSet.as_view({ - 'get': 'retrieve', - }) + view = XFormViewSet.as_view( + { + "get": "retrieve", + } + ) - request = self.factory.get('/', **self.extra) - response = view(request, pk=self.xform.pk, format='csv') + request = self.factory.get("/", **self.extra) + response = view(request, pk=self.xform.pk, format="csv") self.assertEqual(response.status_code, 200) - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) self._create_dataview() - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) - response = view(request, pk=self.data_view.pk, format='csv') + request = self.factory.get("/", **self.extra) + response = view(request, pk=self.data_view.pk, format="csv") self.assertEqual(response.status_code, 200) - self.assertEquals(count + 2, Export.objects.all().count()) + self.assertEqual(count + 2, Export.objects.all().count()) headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], 'application/csv') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/csv") + content_disposition = headers["Content-Disposition"] filename = filename_from_disposition(content_disposition) - basename, ext = os.path.splitext(filename) - self.assertEqual(ext, '.csv') + _basename, ext = os.path.splitext(filename) + self.assertEqual(ext, ".csv") content = get_response_content(response) # count csv headers and ensure they are three - self.assertEqual(len(content.split('\n')[0].split(',')), 3) + self.assertEqual(len(content.split("\n")[0].split(",")), 3) def test_matches_parent(self): self._create_dataview() self.assertFalse(self.data_view.matches_parent) - columns = [u'name', u'age', u'gender', u'photo', u'date', u'location', - u'pizza_fan', u'pizza_hater', u'pizza_type', - u'favorite_toppings', u'test_location2.latitude', - u'test_location2.longitude', u'test_location.precision', - u'test_location2.precision', u'test_location.altitude', - u'test_location.latitude', u'test_location2.altitude', - u'test_location.longitude', u'thanks', u'a_group', - u'a_group/grouped', u'a_group/a_text', u'start_time', - u'end_time', u'today', u'imei', u'phonenumber', - 'meta', 'meta/instanceID'] + columns = [ + "name", + "age", + "gender", + "photo", + "date", + "location", + "pizza_fan", + "pizza_hater", + "pizza_type", + "favorite_toppings", + "test_location2.latitude", + "test_location2.longitude", + "test_location.precision", + "test_location2.precision", + "test_location.altitude", + "test_location.latitude", + "test_location2.altitude", + "test_location.longitude", + "thanks", + "a_group", + "a_group/grouped", + "a_group/a_text", + "start_time", + "end_time", + "today", + "imei", + "phonenumber", + "meta", + "meta/instanceID", + ] data = { - 'name': "My DataView2", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': json.dumps(columns), - 'query': '[{"column":"age","filter":">","value":"20"}]' + "name": "My DataView2", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": json.dumps(columns), + "query": '[{"column":"age","filter":">","value":"20"}]', } self._create_dataview(data) self.assertTrue(self.data_view.matches_parent) + # pylint: disable=invalid-name def test_dataview_create_data_filter_invalid_date(self): - invalid_query = '[{"column":"_submission_time",' \ - '"filter":">","value":"30/06/2015"}]' + invalid_query = ( + '[{"column":"_submission_time","filter":">","value":"30/06/2015"}]' + ) data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "gender", "_submission_time"]', - 'query': invalid_query + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "gender", "_submission_time"]', + "query": invalid_query, } - view = DataViewViewSet.as_view({ - 'get': 'data', - 'post': 'create', - 'patch': 'partial_update' - }) + view = DataViewViewSet.as_view( + {"get": "data", "post": "create", "patch": "partial_update"} + ) - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = view(request) # Confirm you cannot create an invalid dataview - self.assertEquals(response.status_code, 400) + self.assertEqual(response.status_code, 400) + # pylint: disable=invalid-name def test_dataview_update_data_filter_invalid_date(self): - invalid_query = '[{"column":"_submission_time",' \ - '"filter":">","value":"30/06/2015"}]' + invalid_query = ( + '[{"column":"_submission_time","filter":">","value":"30/06/2015"}]' + ) self._create_dataview() - data = {'query': invalid_query} - request = self.factory.patch('/', data=data, **self.extra) + data = {"query": invalid_query} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, pk=self.data_view.pk) # Confirm you cannot update an invalid dataview - self.assertEquals(response.status_code, 400) + self.assertEqual(response.status_code, 400) + # pylint: disable=invalid-name def test_dataview_serializer_exception(self): invalid_query = [ - {"column": "_submission_time", - "filter": ">", - "value": "30/06/2015"} + {"column": "_submission_time", "filter": ">", "value": "30/06/2015"} ] self._create_dataview() self.data_view.query = invalid_query self.data_view.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 400) + self.assertEqual(response.status_code, 400) + # pylint: disable=invalid-name def test_dataview_notes_added_to_data(self): # Create note - view = NoteViewSet.as_view({ - 'post': 'create' - }) - comment = u"Dataview note" - note = {'note': comment} - data_id = self.xform.instances.all().order_by('pk')[0].pk - note['instance'] = data_id - request = self.factory.post('/', data=note, **self.extra) + view = NoteViewSet.as_view({"post": "create"}) + comment = "Dataview note" + note = {"note": comment} + data_id = self.xform.instances.all().order_by("pk")[0].pk + note["instance"] = data_id + request = self.factory.post("/", data=note, **self.extra) self.assertTrue(self.xform.instances.count()) response = view(request) self.assertEqual(response.status_code, 201) # Get dataview with added notes data = { - 'name': "My Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["age"]', + "name": "My Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["age"]', } self._create_dataview(data=data) - view = DataViewViewSet.as_view({ - 'get': 'data' - }) - request = self.factory.get('/', **self.extra) + view = DataViewViewSet.as_view({"get": "data"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 8) - data_with_notes = next(( - d for d in response.data if d["_id"] == data_id)) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 8) + data_with_notes = next((d for d in response.data if d["_id"] == data_id)) self.assertIn("_notes", data_with_notes) - self.assertEquals([{'created_by': self.user.id, - 'id': 1, - 'instance_field': None, - 'note': comment, - 'owner': self.user.username}], - data_with_notes["_notes"]) + self.assertEqual( + [ + { + "created_by": self.user.id, + "id": 1, + "instance_field": None, + "note": comment, + "owner": self.user.username, + } + ], + data_with_notes["_notes"], + ) def test_sort_dataview_data(self): self._create_dataview() - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) data = {"sort": '{"age": -1}'} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertTrue( - self.is_sorted_desc([r.get("age") for r in response.data])) + self.assertEqual(response.status_code, 200) + self.assertTrue(self.is_sorted_desc([r.get("age") for r in response.data])) def test_invalid_date_filter(self): - view = DataViewViewSet.as_view({ - 'get': 'retrieve', - 'post': 'create', - }) + view = DataViewViewSet.as_view( + { + "get": "retrieve", + "post": "create", + } + ) data = { - 'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[{"column":"_submission_time","filter":">",' - '"value":"26-01-2016"}]' + "name": "My DataView", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender"]', + "query": '[{"column":"_submission_time","filter":">",' + '"value":"26-01-2016"}]', } - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(response.data, - { - u'non_field_errors': - [u'Date value in _submission_time should be' - u' yyyy-mm-ddThh:m:s or yyyy-mm-dd'] - }) + self.assertEqual(response.status_code, 400) + self.assertEqual( + response.data, + { + "non_field_errors": [ + "Date value in _submission_time should be" + " yyyy-mm-ddThh:m:s or yyyy-mm-dd" + ] + }, + ) data = { - 'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[{"column":"_submission_time","filter":">",' - '"value":"26/01/2016"}]' + "name": "My DataView", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender"]', + "query": '[{"column":"_submission_time","filter":">",' + '"value":"26/01/2016"}]', } - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(response.data, - { - u'non_field_errors': - [u'Date value in _submission_time should be' - u' yyyy-mm-ddThh:m:s or yyyy-mm-dd'] - }) + self.assertEqual(response.status_code, 400) + self.assertEqual( + response.data, + { + "non_field_errors": [ + "Date value in _submission_time should be" + " yyyy-mm-ddThh:m:s or yyyy-mm-dd" + ] + }, + ) data = { - 'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[{"column":"_submission_time","filter":">",' - '"value":"2016-01-16T00:00:00"}]' + "name": "My DataView", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender"]', + "query": '[{"column":"_submission_time","filter":">",' + '"value":"2016-01-16T00:00:00"}]', } - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = view(request) - self.assertEquals(response.status_code, 201) + self.assertEqual(response.status_code, 201) data = { - 'name': "My DataView2", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[{"column":"_submission_time","filter":">",' - '"value":"2016-01-16"}]' + "name": "My DataView2", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender"]', + "query": '[{"column":"_submission_time","filter":">",' + '"value":"2016-01-16"}]', } - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = view(request) - self.assertEquals(response.status_code, 201) + self.assertEqual(response.status_code, 201) def test_search_dataview_data(self): self._create_dataview() - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) data = {"query": "Fred"} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertEqual(1, len(response.data)) - self.assertEqual("Fred", response.data[0].get('name')) + self.assertEqual("Fred", response.data[0].get("name")) data = {"query": '{"age": 22}'} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertEqual(1, len(response.data)) - self.assertEqual(22, response.data[0].get('age')) + self.assertEqual(22, response.data[0].get("age")) data = {"query": '{"age": {"$gte": 30}}'} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertEqual(1, len(response.data)) - self.assertEqual(45, response.data[0].get('age')) + self.assertEqual(45, response.data[0].get("age")) def test_invalid_url_parameters(self): - response = self.client.get('/api/v1/dataviews/css/ona.css/') - self.assertEquals(response.status_code, 404) + response = self.client.get("/api/v1/dataviews/css/ona.css/") + self.assertEqual(response.status_code, 404) + # pylint: disable=invalid-name,too-many-locals @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch('onadata.apps.api.viewsets.dataview_viewset.AsyncResult') + @patch("onadata.apps.api.viewsets.dataview_viewset.AsyncResult") def test_export_xls_dataview_with_date_filter(self, async_result): """ Test dataview export with a date filter. @@ -1416,38 +1537,48 @@ def test_export_xls_dataview_with_date_filter(self, async_result): start_date = datetime(2014, 9, 12, tzinfo=utc) first_datetime = start_date.strftime(MONGO_STRFTIME) second_datetime = start_date + timedelta(days=1, hours=20) - query_str = '{"_submission_time": {"$gte": "'\ - + first_datetime + '", "$lte": "'\ - + second_datetime.strftime(MONGO_STRFTIME) + '"}}' - - view = DataViewViewSet.as_view({ - 'get': 'export_async', - }) - - request = self.factory.get('/', data={"format": "xls", - "force_xlsx": 'true', - 'include_labels': 'true', - 'query': query_str}, - **self.extra) + query_str = ( + '{"_submission_time": {"$gte": "' + + first_datetime + + '", "$lte": "' + + second_datetime.strftime(MONGO_STRFTIME) + + '"}}' + ) + + view = DataViewViewSet.as_view( + { + "get": "export_async", + } + ) + + request = self.factory.get( + "/", + data={ + "format": "xls", + "force_xlsx": "true", + "include_labels": "true", + "query": query_str, + }, + **self.extra, + ) response = view(request, pk=self.data_view.pk) self.assertIsNotNone(response.data) self.assertEqual(response.status_code, 202) - self.assertTrue('job_uuid' in response.data) - task_id = response.data.get('job_uuid') + self.assertTrue("job_uuid" in response.data) + task_id = response.data.get("job_uuid") - export_pk = Export.objects.all().order_by('pk').reverse()[0].pk + export_pk = Export.objects.all().order_by("pk").reverse()[0].pk # metaclass for mocking results - job = type('AsyncResultMock', (), - {'state': 'SUCCESS', 'result': export_pk}) + job = type("AsyncResultMock", (), {"state": "SUCCESS", "result": export_pk}) async_result.return_value = job - get_data = {'job_uuid': task_id} - request = self.factory.get('/', data=get_data, **self.extra) + get_data = {"job_uuid": task_id} + request = self.factory.get("/", data=get_data, **self.extra) response = view(request, pk=self.data_view.pk) - self.assertIn('export_url', response.data) + self.assertIn("export_url", response.data) self.assertTrue(async_result.called) self.assertEqual(response.status_code, 202) @@ -1456,7 +1587,7 @@ def test_export_xls_dataview_with_date_filter(self, async_result): workbook = load_workbook(export.full_filepath) sheet_name = workbook.get_sheet_names()[0] main_sheet = workbook.get_sheet_by_name(sheet_name) - self.assertIn('Gender', tuple(main_sheet.values)[1]) + self.assertIn("Gender", tuple(main_sheet.values)[1]) self.assertEqual(len(tuple(main_sheet.values)), 3) def test_csv_export_dataview_date_filter(self): @@ -1468,34 +1599,40 @@ def test_csv_export_dataview_date_filter(self): start_date = datetime(2014, 9, 12, tzinfo=utc) first_datetime = start_date.strftime(MONGO_STRFTIME) second_datetime = start_date + timedelta(days=1, hours=20) - query_str = '{"_submission_time": {"$gte": "'\ - + first_datetime + '", "$lte": "'\ - + second_datetime.strftime(MONGO_STRFTIME) + '"}}' + query_str = ( + '{"_submission_time": {"$gte": "' + + first_datetime + + '", "$lte": "' + + second_datetime.strftime(MONGO_STRFTIME) + + '"}}' + ) count = Export.objects.all().count() - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', data={'query': query_str}, - **self.extra) - response = view(request, pk=self.data_view.pk, format='csv') + request = self.factory.get("/", data={"query": query_str}, **self.extra) + response = view(request, pk=self.data_view.pk, format="csv") self.assertEqual(response.status_code, 200) - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], 'application/csv') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/csv") + content_disposition = headers["Content-Disposition"] filename = filename_from_disposition(content_disposition) - basename, ext = os.path.splitext(filename) - self.assertEqual(ext, '.csv') + _basename, ext = os.path.splitext(filename) + self.assertEqual(ext, ".csv") content = get_response_content(response) - self.assertEqual(content, 'name,age,gender\nDennis Wambua,28,male\n') + self.assertEqual(content, "name,age,gender\nDennis Wambua,28,male\n") + # pylint: disable=too-many-locals @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch('onadata.apps.api.viewsets.dataview_viewset.AsyncResult') + @patch("onadata.apps.api.viewsets.dataview_viewset.AsyncResult") def test_csv_export_async_dataview_date_filter(self, async_result): """ Test dataview csv export async with a date filter. @@ -1505,43 +1642,48 @@ def test_csv_export_async_dataview_date_filter(self, async_result): start_date = datetime(2014, 9, 12, tzinfo=utc) first_datetime = start_date.strftime(MONGO_STRFTIME) second_datetime = start_date + timedelta(days=1, hours=20) - query_str = '{"_submission_time": {"$gte": "'\ - + first_datetime + '", "$lte": "'\ - + second_datetime.strftime(MONGO_STRFTIME) + '"}}' + query_str = ( + '{"_submission_time": {"$gte": "' + + first_datetime + + '", "$lte": "' + + second_datetime.strftime(MONGO_STRFTIME) + + '"}}' + ) count = Export.objects.all().count() - view = DataViewViewSet.as_view({ - 'get': 'export_async', - }) + view = DataViewViewSet.as_view( + { + "get": "export_async", + } + ) - request = self.factory.get('/', data={"format": "csv", - 'query': query_str}, - **self.extra) + request = self.factory.get( + "/", data={"format": "csv", "query": query_str}, **self.extra + ) response = view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 202) self.assertIsNotNone(response.data) - self.assertTrue('job_uuid' in response.data) - task_id = response.data.get('job_uuid') - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertTrue("job_uuid" in response.data) + task_id = response.data.get("job_uuid") + self.assertEqual(count + 1, Export.objects.all().count()) - export_pk = Export.objects.all().order_by('pk').reverse()[0].pk + export_pk = Export.objects.all().order_by("pk").reverse()[0].pk # metaclass for mocking results - job = type('AsyncResultMock', (), - {'state': 'SUCCESS', 'result': export_pk}) + job = type("AsyncResultMock", (), {"state": "SUCCESS", "result": export_pk}) async_result.return_value = job - get_data = {'job_uuid': task_id} - request = self.factory.get('/', data=get_data, **self.extra) + get_data = {"job_uuid": task_id} + request = self.factory.get("/", data=get_data, **self.extra) response = view(request, pk=self.data_view.pk) - self.assertIn('export_url', response.data) + self.assertIn("export_url", response.data) self.assertTrue(async_result.called) self.assertEqual(response.status_code, 202) export = Export.objects.get(task_id=task_id) self.assertTrue(export.is_successful) - with open(export.full_filepath, encoding='utf-8') as csv_file: + with open(export.full_filepath, encoding="utf-8") as csv_file: self.assertEqual( - csv_file.read(), - 'name,age,gender\nDennis Wambua,28,male\n') + csv_file.read(), "name,age,gender\nDennis Wambua,28,male\n" + ) diff --git a/onadata/apps/api/tests/viewsets/test_merged_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_merged_xform_viewset.py index 30d4c794d0..083026b8a5 100644 --- a/onadata/apps/api/tests/viewsets/test_merged_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_merged_xform_viewset.py @@ -105,7 +105,7 @@ def _create_merged_dataset(self, geo=False): 'name': 'Merged Dataset', 'project': - "http://testserver/api/v1/projects/%s" % self.project.pk, + f"http://testserver/api/v1/projects/{self.project.pk}", } # anonymous user request = self.factory.post('/', data=data) @@ -554,14 +554,14 @@ def test_rest_service(self): request = self.factory.post('/', data=post_data, **self.extra) response = view(request) - self.assertEquals(response.status_code, 201) - self.assertEquals(count + 3, RestService.objects.count()) + self.assertEqual(response.status_code, 201) + self.assertEqual(count + 3, RestService.objects.count()) # deleting the service for a merged xform deletes the same service from # the individual forms as well. service = RestService.objects.get(xform=xform) service.delete() - self.assertEquals(count, RestService.objects.count()) + self.assertEqual(count, RestService.objects.count()) def test_md_has_deleted_xforms(self): """ @@ -584,7 +584,7 @@ def test_md_has_deleted_xforms(self): 'name': 'Merged Dataset', 'project': - "http://testserver/api/v1/projects/%s" % self.project.pk, + f"http://testserver/api/v1/projects/{self.project.pk}", } request = self.factory.post('/', data=data, **self.extra) @@ -614,7 +614,7 @@ def test_md_has_no_matching_fields(self): 'name': 'Merged Dataset', 'project': - "http://testserver/api/v1/projects/%s" % self.project.pk, + f"http://testserver/api/v1/projects/{self.project.pk}", } request = self.factory.post('/', data=data, **self.extra) @@ -688,7 +688,7 @@ def test_xform_has_uncommon_reference(self): 'name': 'Merged Dataset', 'project': - "http://testserver/api/v1/projects/%s" % self.project.pk, + f"http://testserver/api/v1/projects/{self.project.pk}", } request = self.factory.post('/', data=data, **self.extra) diff --git a/onadata/apps/api/tests/viewsets/test_metadata_viewset.py b/onadata/apps/api/tests/viewsets/test_metadata_viewset.py index c13dad89a4..255c0690a7 100644 --- a/onadata/apps/api/tests/viewsets/test_metadata_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_metadata_viewset.py @@ -102,7 +102,7 @@ def test_parse_error_is_raised(self): # Duplicate upload response = self._add_form_metadata(self.xform, data_type, self.data_value, self.path, False) - self.assertEquals(response.status_code, 400) + self.assertEqual(response.status_code, 400) self.assertIn(UNIQUE_TOGETHER_ERROR, response.data) def test_forms_endpoint_with_metadata(self): @@ -389,14 +389,14 @@ def test_should_return_both_xform_and_project_metadata(self): request = self.factory.get("/", **self.extra) response = view(request) - self.assertEquals(MetaData.objects.count(), expected_metadata_count) + self.assertEqual(MetaData.objects.count(), expected_metadata_count) for record in response.data: if record.get("xform"): - self.assertEquals(record.get('xform'), self.xform.id) + self.assertEqual(record.get('xform'), self.xform.id) self.assertIsNone(record.get('project')) else: - self.assertEquals(record.get('project'), self.project.id) + self.assertEqual(record.get('project'), self.project.id) self.assertIsNone(record.get('xform')) def test_should_only_return_xform_metadata(self): @@ -594,5 +594,5 @@ def test_unique_submission_review_metadata(self): request = self.factory.post('/', data, **self.extra) d_response = view(request) - self.assertEquals(d_response.status_code, 400) + self.assertEqual(d_response.status_code, 400) self.assertIn(UNIQUE_TOGETHER_ERROR, d_response.data) diff --git a/onadata/apps/api/tests/viewsets/test_note_viewset.py b/onadata/apps/api/tests/viewsets/test_note_viewset.py index f73a227543..9b6cba82d1 100644 --- a/onadata/apps/api/tests/viewsets/test_note_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_note_viewset.py @@ -17,39 +17,38 @@ class TestNoteViewSet(TestBase): """ Test NoteViewSet """ + def setUp(self): super(TestNoteViewSet, self).setUp() self._create_user_and_login() self._publish_transportation_form() self._make_submissions() - self.view = NoteViewSet.as_view({ - 'get': 'list', - 'post': 'create', - 'delete': 'destroy' - }) + self.view = NoteViewSet.as_view( + {"get": "list", "post": "create", "delete": "destroy"} + ) self.factory = RequestFactory() - self.extra = {'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} @property def _first_xform_instance(self): - return self.xform.instances.all().order_by('pk')[0] + return self.xform.instances.all().order_by("pk")[0] def _add_notes_to_data_point(self): # add a note to a specific data point - note = {'note': u"Road Warrior"} + note = {"note": "Road Warrior"} dataid = self._first_xform_instance.pk - note['instance'] = dataid - request = self.factory.post('/', data=note, **self.extra) + note["instance"] = dataid + request = self.factory.post("/", data=note, **self.extra) self.assertTrue(self.xform.instances.count()) response = self.view(request) self.assertEqual(response.status_code, 201) - self.pk = response.data['id'] - note['id'] = self.pk + self.pk = response.data["id"] + note["id"] = self.pk self.note = note def test_note_list(self): self._add_notes_to_data_point() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertTrue(len(response.data) > 0) @@ -57,29 +56,29 @@ def test_note_list(self): def test_note_get(self): self._add_notes_to_data_point() - view = NoteViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/', **self.extra) + view = NoteViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.pk) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['owner'], self.user.username) + self.assertEqual(response.data["owner"], self.user.username) self.assertDictContainsSubset(self.note, response.data) def test_get_note_for_specific_instance(self): self._add_notes_to_data_point() - view = NoteViewSet.as_view({'get': 'retrieve'}) + view = NoteViewSet.as_view({"get": "retrieve"}) instance = self.xform.instances.first() query_params = {"instance": instance.id} - request = self.factory.get('/', data=query_params, **self.extra) + request = self.factory.get("/", data=query_params, **self.extra) response = view(request, pk=self.pk) self.assertEqual(response.status_code, 200) self.assertDictContainsSubset(self.note, response.data) second_instance = self.xform.instances.last() query_params = {"instance": second_instance.id} - request = self.factory.get('/', data=query_params, **self.extra) + request = self.factory.get("/", data=query_params, **self.extra) response = view(request, pk=self.pk) self.assertEqual(response.status_code, 200) @@ -87,18 +86,18 @@ def test_get_note_for_specific_instance(self): def test_add_notes_to_data_point(self): self._add_notes_to_data_point() - self.assertEquals(len(self._first_xform_instance.json["_notes"]), 1) + self.assertEqual(len(self._first_xform_instance.json["_notes"]), 1) def test_other_user_notes_access(self): - self._create_user_and_login('lilly', '1234') - extra = {'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} - note = {'note': u"Road Warrior"} + self._create_user_and_login("lilly", "1234") + extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} + note = {"note": "Road Warrior"} dataid = self.xform.instances.first().pk - note['instance'] = dataid + note["instance"] = dataid # Other user 'lilly' should not be able to create notes # to xform instance owned by 'bob' - request = self.factory.post('/', data=note) + request = self.factory.post("/", data=note) self.assertTrue(self.xform.instances.count()) response = self.view(request) self.assertEqual(response.status_code, 401) @@ -107,44 +106,44 @@ def test_other_user_notes_access(self): self._add_notes_to_data_point() # access to /notes endpoint,should be empty list - request = self.factory.get('/', **extra) + request = self.factory.get("/", **extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) # Other user 'lilly' sees an empty list when accessing bob's notes - view = NoteViewSet.as_view({'get': 'retrieve'}) + view = NoteViewSet.as_view({"get": "retrieve"}) query_params = {"instance": dataid} - request = self.factory.get('/', data=query_params, **extra) + request = self.factory.get("/", data=query_params, **extra) response = view(request, pk=self.pk) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) def test_collaborator_with_readonly_permission_can_add_comment(self): - self._create_user_and_login('lilly', '1234') - extra = {'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} + self._create_user_and_login("lilly", "1234") + extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} # save some notes self._add_notes_to_data_point() # post note to submission as lilly without permissions - note = {'note': u"Road Warrior"} + note = {"note": "Road Warrior"} dataid = self._first_xform_instance.pk - note['instance'] = dataid - request = self.factory.post('/', data=note) + note["instance"] = dataid + request = self.factory.post("/", data=note) self.assertTrue(self.xform.instances.count()) response = self.view(request) self.assertEqual(response.status_code, 401) # post note to submission with permissions to form - assign_perm('view_xform', self.user, self._first_xform_instance.xform) + assign_perm("view_xform", self.user, self._first_xform_instance.xform) - note = {'note': u"Road Warrior"} + note = {"note": "Road Warrior"} dataid = self._first_xform_instance.pk - note['instance'] = dataid - request = self.factory.post('/', data=note, **extra) + note["instance"] = dataid + request = self.factory.post("/", data=note, **extra) self.assertTrue(self.xform.instances.count()) response = self.view(request) @@ -152,48 +151,40 @@ def test_collaborator_with_readonly_permission_can_add_comment(self): def test_delete_note(self): self._add_notes_to_data_point() - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = self.view(request, pk=self.pk) self.assertEqual(response.status_code, 204) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) - self.assertEquals(response.data, []) + self.assertEqual(response.data, []) def test_question_level_notes(self): field = "transport" dataid = self.xform.instances.all()[0].pk - note = { - 'note': "Road Warrior", - 'instance': dataid, - 'instance_field': field - } - request = self.factory.post('/', data=note, **self.extra) + note = {"note": "Road Warrior", "instance": dataid, "instance_field": field} + request = self.factory.post("/", data=note, **self.extra) self.assertTrue(self.xform.instances.count()) response = self.view(request) self.assertEqual(response.status_code, 201) - instance = self.xform.instances.all()[0] - self.assertEquals(len(instance.json["_notes"]), 1) + instance = self.xform.instances.get(pk=dataid) + self.assertEqual(len(instance.json["_notes"]), 1, instance.json) note = instance.json["_notes"][0] - self.assertEquals(note['instance_field'], field) + self.assertEqual(note["instance_field"], field) def test_only_add_question_notes_to_existing_fields(self): field = "bla" dataid = self.xform.instances.all()[0].pk - note = { - 'note': "Road Warrior", - 'instance': dataid, - 'instance_field': field - } - request = self.factory.post('/', data=note, **self.extra) + note = {"note": "Road Warrior", "instance": dataid, "instance_field": field} + request = self.factory.post("/", data=note, **self.extra) self.assertTrue(self.xform.instances.count()) response = self.view(request) self.assertEqual(response.status_code, 400) instance = self.xform.instances.all()[0] - self.assertEquals(len(instance.json["_notes"]), 0) + self.assertEqual(len(instance.json["_notes"]), 0) def test_csv_export_form_w_notes(self): """ @@ -208,26 +199,34 @@ def test_csv_export_form_w_notes(self): instance.save() instance.parsed_instance.save() - view = XFormViewSet.as_view({'get': 'retrieve'}) + view = XFormViewSet.as_view({"get": "retrieve"}) - request = self.factory.get('/', **self.extra) - response = view(request, pk=self.xform.pk, format='csv') + request = self.factory.get("/", **self.extra) + response = view(request, pk=self.xform.pk, format="csv") self.assertTrue(response.status_code, 200) - test_file_path = os.path.join(settings.PROJECT_ROOT, 'apps', 'viewer', - 'tests', 'fixtures', - 'transportation_w_notes.csv') + test_file_path = os.path.join( + settings.PROJECT_ROOT, + "apps", + "viewer", + "tests", + "fixtures", + "transportation_w_notes.csv", + ) self._test_csv_response(response, test_file_path) def test_attribute_error_bug(self): """NoteSerializer: Should not raise AttributeError exeption""" - note = Note(note='Hello', instance=self._first_xform_instance) + note = Note(note="Hello", instance=self._first_xform_instance) note.save() data = NoteSerializer(note).data - self.assertDictContainsSubset({ - 'created_by': None, - 'note': u'Hello', - 'instance': note.instance_id, - 'owner': None - }, data) + self.assertDictContainsSubset( + { + "created_by": None, + "note": "Hello", + "instance": note.instance_id, + "owner": None, + }, + data, + ) diff --git a/onadata/apps/api/tests/viewsets/test_ona_api.py b/onadata/apps/api/tests/viewsets/test_ona_api.py index 98a64f40ab..84f7e2b38e 100644 --- a/onadata/apps/api/tests/viewsets/test_ona_api.py +++ b/onadata/apps/api/tests/viewsets/test_ona_api.py @@ -18,7 +18,7 @@ def test_number_of_v1_viewsets(self): request = self.factory.get(path) request.resolver_match = resolve(path) response = view(request) - self.assertEquals(len(response.data), 29) + self.assertEqual(len(response.data), 29) def test_number_of_v2_viewsets(self): ''' @@ -30,4 +30,4 @@ def test_number_of_v2_viewsets(self): request = self.factory.get(path) request.resolver_match = resolve(path) response = view(request) - self.assertEquals(len(response.data), 1) + self.assertEqual(len(response.data), 1) diff --git a/onadata/apps/api/tests/viewsets/test_organization_profile_viewset.py b/onadata/apps/api/tests/viewsets/test_organization_profile_viewset.py index 2ed4e777ba..38f7d14837 100644 --- a/onadata/apps/api/tests/viewsets/test_organization_profile_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_organization_profile_viewset.py @@ -1,33 +1,46 @@ +# -*- coding: utf-8 -*- +""" +Test /orgs API endpoint implementation. +""" import json from builtins import str as text from django.contrib.auth.models import User from django.core.cache import cache + from mock import patch from rest_framework import status from onadata.apps.api.models.organization_profile import OrganizationProfile -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet -from onadata.apps.api.tools import (get_or_create_organization_owners_team, - add_user_to_organization) -from onadata.apps.api.viewsets.organization_profile_viewset import \ - OrganizationProfileViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet +from onadata.apps.api.tools import ( + add_user_to_organization, + get_or_create_organization_owners_team, +) +from onadata.apps.api.viewsets.organization_profile_viewset import ( + OrganizationProfileViewSet, +) from onadata.apps.api.viewsets.project_viewset import ProjectViewSet from onadata.apps.api.viewsets.user_profile_viewset import UserProfileViewSet from onadata.apps.main.models import UserProfile from onadata.libs.permissions import OwnerRole +# pylint: disable=too-many-public-methods class TestOrganizationProfileViewSet(TestAbstractViewSet): + """ + Test /orgs API endpoint implementation. + """ def setUp(self): - super(self.__class__, self).setUp() - self.view = OrganizationProfileViewSet.as_view({ - 'get': 'list', - 'post': 'create', - 'patch': 'partial_update', - }) + super().setUp() + self.view = OrganizationProfileViewSet.as_view( + { + "get": "list", + "post": "create", + "patch": "partial_update", + } + ) def tearDown(self): """ @@ -38,32 +51,32 @@ def tearDown(self): def test_partial_updates(self): self._org_create() - metadata = {u'computer': u'mac'} + metadata = {"computer": "mac"} json_metadata = json.dumps(metadata) - data = {'metadata': json_metadata} - request = self.factory.patch('/', data=data, **self.extra) - response = self.view(request, user='denoinc') - profile = OrganizationProfile.objects.get(name='Dennis') + data = {"metadata": json_metadata} + request = self.factory.patch("/", data=data, **self.extra) + response = self.view(request, user="denoinc") + profile = OrganizationProfile.objects.get(name="Dennis") self.assertEqual(response.status_code, 200) self.assertEqual(profile.metadata, metadata) def test_partial_updates_invalid(self): self._org_create() - data = {'name': "a" * 31} - request = self.factory.patch('/', data=data, **self.extra) - response = self.view(request, user='denoinc') + data = {"name": "a" * 31} + request = self.factory.patch("/", data=data, **self.extra) + response = self.view(request, user="denoinc") self.assertEqual(response.status_code, 400) self.assertEqual( - response.data['name'], - [u'Ensure this field has no more than 30 characters.']) + response.data["name"], ["Ensure this field has no more than 30 characters."] + ) def test_orgs_list(self): self._org_create() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - del self.company_data['metadata'] + del self.company_data["metadata"] self.assertEqual(response.data, [self.company_data]) # inactive organization @@ -76,129 +89,117 @@ def test_orgs_list(self): def test_orgs_list_for_anonymous_user(self): self._org_create() - request = self.factory.get('/') + request = self.factory.get("/") response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) def test_orgs_list_for_authenticated_user(self): self._org_create() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - del self.company_data['metadata'] + del self.company_data["metadata"] self.assertEqual(response.data, [self.company_data]) def test_orgs_list_shared_with_user(self): authenticated_user = self.user user_in_shared_organization, _ = User.objects.get_or_create( - username='the_stalked') + username="the_stalked" + ) UserProfile.objects.get_or_create( - user=user_in_shared_organization, - name=user_in_shared_organization.username + user=user_in_shared_organization, name=user_in_shared_organization.username ) - unshared_organization, _ = User.objects.get_or_create( - username='NotShared') - unshared_organization_profile, _ = OrganizationProfile\ - .objects.get_or_create( - user=unshared_organization, - creator=authenticated_user) + unshared_organization, _ = User.objects.get_or_create(username="NotShared") + unshared_organization_profile, _ = OrganizationProfile.objects.get_or_create( + user=unshared_organization, creator=authenticated_user + ) - add_user_to_organization(unshared_organization_profile, - authenticated_user) + add_user_to_organization(unshared_organization_profile, authenticated_user) - shared_organization, _ = User.objects.get_or_create(username='Shared') - shared_organization_profile, _ = OrganizationProfile\ - .objects.get_or_create( - user=shared_organization, - creator=user_in_shared_organization) + shared_organization, _ = User.objects.get_or_create(username="Shared") + shared_organization_profile, _ = OrganizationProfile.objects.get_or_create( + user=shared_organization, creator=user_in_shared_organization + ) - add_user_to_organization(shared_organization_profile, - authenticated_user) + add_user_to_organization(shared_organization_profile, authenticated_user) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertTrue(len(response.data), 2) - request = self.factory.get('/', - data={'shared_with': 'the_stalked'}, - **self.extra) + request = self.factory.get( + "/", data={"shared_with": "the_stalked"}, **self.extra + ) response = self.view(request) self.assertEqual(len(response.data), 1) def test_orgs_list_restricted(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'get': 'list' - }) + view = OrganizationProfileViewSet.as_view({"get": "list"}) - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(extra_post_data=alice_data) - self.assertEqual(self.user.username, 'alice') + self.assertEqual(self.user.username, "alice") - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.data, []) def test_orgs_get(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = OrganizationProfileViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data, {'detail': 'Expected URL keyword argument `user`.'}) - request = self.factory.get('/', **self.extra) - response = view(request, user='denoinc') - self.assertNotEqual(response.get('Cache-Control'), None) + response.data, {"detail": "Expected URL keyword argument `user`."} + ) + request = self.factory.get("/", **self.extra) + response = view(request, user="denoinc") + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.company_data) - self.assertIn('users', list(response.data)) - for user in response.data['users']: - self.assertEqual(user['role'], 'owner') - self.assertTrue(isinstance(user['user'], text)) + self.assertIn("users", list(response.data)) + for user in response.data["users"]: + self.assertEqual(user["role"], "owner") + self.assertTrue(isinstance(user["user"], text)) def test_orgs_get_not_creator(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'get': 'retrieve' - }) - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + view = OrganizationProfileViewSet.as_view({"get": "retrieve"}) + alice_data = {"username": "alice", "email": "alice@localhost.com"} previous_user = self.user self._login_user_and_profile(extra_post_data=alice_data) - self.assertEqual(self.user.username, 'alice') + self.assertEqual(self.user.username, "alice") self.assertNotEqual(previous_user, self.user) - request = self.factory.get('/', **self.extra) - response = view(request, user='denoinc') - self.assertNotEqual(response.get('Cache-Control'), None) + request = self.factory.get("/", **self.extra) + response = view(request, user="denoinc") + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.company_data) - self.assertIn('users', list(response.data)) - for user in response.data['users']: - self.assertEqual(user['role'], 'owner') - self.assertTrue(isinstance(user['user'], text)) + self.assertIn("users", list(response.data)) + for user in response.data["users"]: + self.assertEqual(user["role"], "owner") + self.assertTrue(isinstance(user["user"], text)) def test_orgs_get_anon(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/') - response = view(request, user='denoinc') - self.assertNotEqual(response.get('Cache-Control'), None) + view = OrganizationProfileViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/") + response = view(request, user="denoinc") + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.company_data) - self.assertIn('users', list(response.data)) - for user in response.data['users']: - self.assertEqual(user['role'], 'owner') - self.assertTrue(isinstance(user['user'], text)) + self.assertIn("users", list(response.data)) + for user in response.data["users"]: + self.assertEqual(user["role"], "owner") + self.assertTrue(isinstance(user["user"], text)) def test_orgs_create(self): self._org_create() @@ -206,161 +207,149 @@ def test_orgs_create(self): def test_orgs_create_without_name(self): data = { - 'org': u'denoinc', - 'city': u'Denoville', - 'country': u'US', - 'home_page': u'deno.com', - 'twitter': u'denoinc', - 'description': u'', - 'address': u'', - 'phonenumber': u'', - 'require_auth': False, + "org": "denoinc", + "city": "Denoville", + "country": "US", + "home_page": "deno.com", + "twitter": "denoinc", + "description": "", + "address": "", + "phonenumber": "", + "require_auth": False, } request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = self.view(request) - self.assertEqual(response.data, {'name': [u'This field is required.']}) + self.assertEqual(response.data, {"name": ["This field is required."]}) def test_org_create_with_anonymous_user(self): data = { - 'name': u'denoinc', - 'org': u'denoinc', - 'city': u'Denoville', - 'country': u'US', - 'home_page': u'deno.com', - 'twitter': u'denoinc', - 'description': u'', - 'address': u'', - 'phonenumber': u'', - 'require_auth': False, + "name": "denoinc", + "org": "denoinc", + "city": "Denoville", + "country": "US", + "home_page": "deno.com", + "twitter": "denoinc", + "description": "", + "address": "", + "phonenumber": "", + "require_auth": False, } request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json") + "/", data=json.dumps(data), content_type="application/json" + ) response = self.view(request) - self.assertEquals(response.status_code, 401) + self.assertEqual(response.status_code, 401) def test_orgs_members_list(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'get': 'members' - }) + view = OrganizationProfileViewSet.as_view({"get": "members"}) - request = self.factory.get('/', **self.extra) - response = view(request, user='denoinc') + request = self.factory.get("/", **self.extra) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) - self.assertEqual(response.data, [u'denoinc']) + self.assertNotEqual(response.get("Cache-Control"), None) + self.assertEqual(response.data, ["denoinc"]) def test_add_members_to_org_username_required(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) - request = self.factory.post('/', data={}, **self.extra) - response = view(request, user='denoinc') + view = OrganizationProfileViewSet.as_view({"post": "members"}) + request = self.factory.post("/", data={}, **self.extra) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - {u'username': [u"This field may not be null."]}) + self.assertEqual(response.data, {"username": ["This field may not be null."]}) def test_add_members_to_org_user_does_not_exist(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) - data = {'username': 'aboy'} + view = OrganizationProfileViewSet.as_view({"post": "members"}) + data = {"username": "aboy"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - {u'username': [u"User 'aboy' does not exist."]}) + self.assertEqual(response.data, {"username": ["User 'aboy' does not exist."]}) def test_add_members_to_org(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) + view = OrganizationProfileViewSet.as_view({"post": "members"}) - self.profile_data['username'] = 'aboy' - self.profile_data['email'] = 'aboy@org.com' + self.profile_data["username"] = "aboy" + self.profile_data["email"] = "aboy@org.com" self._create_user_profile() - data = {'username': 'aboy'} + data = {"username": "aboy"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', u'aboy'])) + self.assertEqual(set(response.data), set(["denoinc", "aboy"])) def test_add_members_to_org_user_org_account(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) + view = OrganizationProfileViewSet.as_view({"post": "members"}) - username = 'second_inc' + username = "second_inc" # Create second org - org_data = {'org': username} + org_data = {"org": username} self._org_create(org_data=org_data) - data = {'username': username} + data = {"username": username} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - {'username': [u'Cannot add org account `second_inc` ' - u'as member.']}) + self.assertEqual( + response.data, + {"username": ["Cannot add org account `second_inc` " "as member."]}, + ) def test_member_sees_orgs_added_to(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'get': 'list', - 'post': 'members' - }) - - member = 'aboy' - cur_username = self.profile_data['username'] - self.profile_data['username'] = member + view = OrganizationProfileViewSet.as_view({"get": "list", "post": "members"}) + + member = "aboy" + cur_username = self.profile_data["username"] + self.profile_data["username"] = member self._login_user_and_profile() - self.profile_data['username'] = cur_username + self.profile_data["username"] = cur_username self._login_user_and_profile() - data = {'username': member} + data = {"username": member} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) - response = view(request, user='denoinc') + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', u'aboy'])) + self.assertEqual(set(response.data), set(["denoinc", "aboy"])) - self.profile_data['username'] = member + self.profile_data["username"] = member self._login_user_and_profile() expected_data = self.company_data - expected_data['users'].append({ - 'first_name': u'Bob', - 'last_name': u'erama', - 'role': 'member', - 'user': member, - 'gravatar': self.user.profile.gravatar, - }) - del expected_data['metadata'] - - request = self.factory.get('/', **self.extra) + expected_data["users"].append( + { + "first_name": "Bob", + "last_name": "erama", + "role": "member", + "user": member, + "gravatar": self.user.profile.gravatar, + } + ) + del expected_data["metadata"] + + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) response_data = dict(response.data[0]) - returned_users = response_data.pop('users') - expected_users = expected_data.pop('users') + returned_users = response_data.pop("users") + expected_users = expected_data.pop("users") self.assertDictEqual(response_data, expected_data) for user in expected_users: self.assertIn(user, returned_users) @@ -368,419 +357,401 @@ def test_member_sees_orgs_added_to(self): def test_role_for_org_non_owner(self): # creating org with member self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'get': 'retrieve', - 'post': 'members' - }) + view = OrganizationProfileViewSet.as_view( + {"get": "retrieve", "post": "members"} + ) - self.profile_data['username'] = "aboy" + self.profile_data["username"] = "aboy" self._create_user_profile() - data = {'username': 'aboy'} - user_role = 'member' + data = {"username": "aboy"} + user_role = "member" request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) - response = view(request, user='denoinc') + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) + response = view(request, user="denoinc") # getting profile - request = self.factory.get('/', **self.extra) - response = view(request, user='denoinc') + request = self.factory.get("/", **self.extra) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - self.assertIn('users', list(response.data)) - - for user in response.data['users']: - username = user['user'] - role = user['role'] - expected_role = 'owner' if username == 'denoinc' or \ - username == self.user.username else user_role + self.assertIn("users", list(response.data)) + + for user in response.data["users"]: + username = user["user"] + role = user["role"] + expected_role = ( + "owner" + if username == "denoinc" or username == self.user.username + else user_role + ) self.assertEqual(role, expected_role) def test_add_members_to_org_with_anonymous_user(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) + view = OrganizationProfileViewSet.as_view({"post": "members"}) - self._create_user_profile(extra_post_data={'username': 'aboy'}) - data = {'username': 'aboy'} + self._create_user_profile(extra_post_data={"username": "aboy"}) + data = {"username": "aboy"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json") + "/", data=json.dumps(data), content_type="application/json" + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 401) - self.assertNotEquals(set(response.data), set([u'denoinc', u'aboy'])) + self.assertNotEqual(set(response.data), set(["denoinc", "aboy"])) def test_add_members_to_org_with_non_member_user(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) + view = OrganizationProfileViewSet.as_view({"post": "members"}) - self._create_user_profile(extra_post_data={'username': 'aboy'}) - data = {'username': 'aboy'} + self._create_user_profile(extra_post_data={"username": "aboy"}) + data = {"username": "aboy"} previous_user = self.user - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(extra_post_data=alice_data) - self.assertEqual(self.user.username, 'alice') - self.assertNotEqual(previous_user, self.user) + self.assertEqual(self.user.username, "alice") + self.assertNotEqual(previous_user, self.user) request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 404) - self.assertNotEqual(set(response.data), set([u'denoinc', u'aboy'])) + self.assertNotEqual(set(response.data), set(["denoinc", "aboy"])) def test_remove_members_from_org(self): self._org_create() - newname = 'aboy' - view = OrganizationProfileViewSet.as_view({ - 'post': 'members', - 'delete': 'members' - }) + newname = "aboy" + view = OrganizationProfileViewSet.as_view( + {"post": "members", "delete": "members"} + ) - self._create_user_profile(extra_post_data={'username': newname}) + self._create_user_profile(extra_post_data={"username": newname}) - data = {'username': newname} + data = {"username": newname} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', newname])) + self.assertEqual(set(response.data), set(["denoinc", newname])) request = self.factory.delete( - '/', json.dumps(data), - content_type="application/json", **self.extra) + "/", json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data, [u'denoinc']) + self.assertEqual(response.data, ["denoinc"]) - newname = 'aboy2' - self._create_user_profile(extra_post_data={'username': newname}) + newname = "aboy2" + self._create_user_profile(extra_post_data={"username": newname}) - data = {'username': newname} + data = {"username": newname} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', newname])) + self.assertEqual(set(response.data), set(["denoinc", newname])) - request = self.factory.delete( - '/?username={}'.format(newname), **self.extra) + request = self.factory.delete("/?username={}".format(newname), **self.extra) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data, [u'denoinc']) + self.assertEqual(response.data, ["denoinc"]) # Removes users from org projects. # Create a project - project_data = { - 'owner': self.company_data['user'] - } + project_data = {"owner": self.company_data["user"]} self._project_create(project_data) # Create alice - alice = 'alice' - self._create_user_profile(extra_post_data={'username': alice}) - alice_data = {'username': alice, - 'role': 'owner'} + alice = "alice" + self._create_user_profile(extra_post_data={"username": alice}) + alice_data = {"username": alice, "role": "owner"} request = self.factory.post( - '/', data=json.dumps(alice_data), - content_type="application/json", **self.extra) - response = view(request, user='denoinc') + "/", + data=json.dumps(alice_data), + content_type="application/json", + **self.extra + ) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) # alice is in project - projectView = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + projectView = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = projectView(request, pk=self.project.pk) - project_users = response.data.get('users') - users_in_users = [user['user'] for user in project_users] + project_users = response.data.get("users") + users_in_users = [user["user"] for user in project_users] self.assertIn(alice, users_in_users) # remove alice from org - request = self.factory.delete( - '/?username={}'.format(alice), **self.extra) + request = self.factory.delete("/?username={}".format(alice), **self.extra) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - self.assertNotIn(response.data, [u'alice']) + self.assertNotIn(response.data, ["alice"]) # alice is also removed from project - projectView = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + projectView = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = projectView(request, pk=self.project.pk) - project_users = response.data.get('users') - users_in_users = [user['user'] for user in project_users] + project_users = response.data.get("users") + users_in_users = [user["user"] for user in project_users] self.assertNotIn(alice, users_in_users) def test_orgs_create_with_mixed_case(self): data = { - 'name': u'denoinc', - 'org': u'DenoINC', - 'city': u'Denoville', - 'country': u'US', - 'home_page': u'deno.com', - 'twitter': u'denoinc', - 'description': u'', - 'address': u'', - 'phonenumber': u'', - 'require_auth': False, + "name": "denoinc", + "org": "DenoINC", + "city": "Denoville", + "country": "US", + "home_page": "deno.com", + "twitter": "denoinc", + "description": "", + "address": "", + "phonenumber": "", + "require_auth": False, } request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - data['org'] = 'denoinc' + data["org"] = "denoinc" request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 400) - self.assertIn("Organization %s already exists." % data['org'], - response.data['org']) + self.assertIn( + "Organization %s already exists." % data["org"], response.data["org"] + ) def test_publish_xls_form_to_organization_project(self): self._org_create() - project_data = { - 'owner': self.company_data['user'] - } + project_data = {"owner": self.company_data["user"]} self._project_create(project_data) self._publish_xls_form_to_project() self.assertTrue(OwnerRole.user_has_role(self.user, self.xform)) def test_put_change_role(self): self._org_create() - newname = 'aboy' - view = OrganizationProfileViewSet.as_view({ - 'get': 'retrieve', - 'post': 'members', - 'put': 'members' - }) - - self.profile_data['username'] = newname + newname = "aboy" + view = OrganizationProfileViewSet.as_view( + {"get": "retrieve", "post": "members", "put": "members"} + ) + + self.profile_data["username"] = newname self._create_user_profile() - data = {'username': newname} + data = {"username": newname} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(sorted(response.data), sorted([u'denoinc', newname])) + self.assertEqual(sorted(response.data), sorted(["denoinc", newname])) - user_role = 'editor' - data = {'username': newname, 'role': user_role} + user_role = "editor" + data = {"username": newname, "role": user_role} request = self.factory.put( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - self.assertEqual(sorted(response.data), sorted([u'denoinc', newname])) + self.assertEqual(sorted(response.data), sorted(["denoinc", newname])) # getting profile - request = self.factory.get('/', **self.extra) - response = view(request, user='denoinc') + request = self.factory.get("/", **self.extra) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - self.assertIn('users', list(response.data)) - - for user in response.data['users']: - username = user['user'] - role = user['role'] - expected_role = 'owner' if username == 'denoinc' or \ - username == self.user.username else user_role + self.assertIn("users", list(response.data)) + + for user in response.data["users"]: + username = user["user"] + role = user["role"] + expected_role = ( + "owner" + if username == "denoinc" or username == self.user.username + else user_role + ) self.assertEqual(role, expected_role) def test_put_require_role(self): self._org_create() - newname = 'aboy' - view = OrganizationProfileViewSet.as_view({ - 'get': 'retrieve', - 'post': 'members', - 'put': 'members' - }) - - self.profile_data['username'] = newname + newname = "aboy" + view = OrganizationProfileViewSet.as_view( + {"get": "retrieve", "post": "members", "put": "members"} + ) + + self.profile_data["username"] = newname self._create_user_profile() - data = {'username': newname} + data = {"username": newname} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', newname])) + self.assertEqual(set(response.data), set(["denoinc", newname])) - data = {'username': newname} + data = {"username": newname} request = self.factory.put( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 400) def test_put_bad_role(self): self._org_create() - newname = 'aboy' - view = OrganizationProfileViewSet.as_view({ - 'get': 'retrieve', - 'post': 'members', - 'put': 'members' - }) - - self.profile_data['username'] = newname + newname = "aboy" + view = OrganizationProfileViewSet.as_view( + {"get": "retrieve", "post": "members", "put": "members"} + ) + + self.profile_data["username"] = newname self._create_user_profile() - data = {'username': newname} + data = {"username": newname} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', newname])) + self.assertEqual(set(response.data), set(["denoinc", newname])) - data = {'username': newname, 'role': 42} + data = {"username": newname, "role": 42} request = self.factory.put( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 400) - @patch('onadata.libs.serializers.organization_member_serializer.send_mail') + @patch("onadata.libs.serializers.organization_member_serializer.send_mail") def test_add_members_to_org_email(self, mock_email): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) + view = OrganizationProfileViewSet.as_view({"post": "members"}) - self.profile_data['username'] = 'aboy' - self.profile_data['email'] = 'aboy@org.com' + self.profile_data["username"] = "aboy" + self.profile_data["email"] = "aboy@org.com" self._create_user_profile() - data = {'username': 'aboy', - 'email_msg': 'You have been add to denoinc'} + data = {"username": "aboy", "email_msg": "You have been add to denoinc"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) self.assertTrue(mock_email.called) - mock_email.assert_called_with('aboy, You have been added to Dennis' - ' organisation.', - u'You have been add to denoinc', - 'noreply@ona.io', - (u'aboy@org.com',)) - self.assertEqual(set(response.data), set([u'denoinc', u'aboy'])) - - @patch('onadata.libs.serializers.organization_member_serializer.send_mail') + mock_email.assert_called_with( + "aboy, You have been added to Dennis" " organisation.", + "You have been add to denoinc", + "noreply@ona.io", + ("aboy@org.com",), + ) + self.assertEqual(set(response.data), set(["denoinc", "aboy"])) + + @patch("onadata.libs.serializers.organization_member_serializer.send_mail") def test_add_members_to_org_email_custom_subj(self, mock_email): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) + view = OrganizationProfileViewSet.as_view({"post": "members"}) - self.profile_data['username'] = 'aboy' - self.profile_data['email'] = 'aboy@org.com' + self.profile_data["username"] = "aboy" + self.profile_data["email"] = "aboy@org.com" self._create_user_profile() - data = {'username': 'aboy', - 'email_msg': 'You have been add to denoinc', - 'email_subject': 'Your are made'} + data = { + "username": "aboy", + "email_msg": "You have been add to denoinc", + "email_subject": "Your are made", + } request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) self.assertTrue(mock_email.called) - mock_email.assert_called_with('Your are made', - u'You have been add to denoinc', - 'noreply@ona.io', - (u'aboy@org.com',)) - self.assertEqual(set(response.data), set([u'denoinc', u'aboy'])) + mock_email.assert_called_with( + "Your are made", + "You have been add to denoinc", + "noreply@ona.io", + ("aboy@org.com",), + ) + self.assertEqual(set(response.data), set(["denoinc", "aboy"])) def test_add_members_to_org_with_role(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members', - 'get': 'retrieve' - }) + view = OrganizationProfileViewSet.as_view( + {"post": "members", "get": "retrieve"} + ) - self.profile_data['username'] = "aboy" + self.profile_data["username"] = "aboy" self._create_user_profile() - data = {'username': 'aboy', - 'role': 'editor'} + data = {"username": "aboy", "role": "editor"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', u'aboy'])) + self.assertEqual(set(response.data), set(["denoinc", "aboy"])) # getting profile - request = self.factory.get('/', **self.extra) - response = view(request, user='denoinc') + request = self.factory.get("/", **self.extra) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['users'][2]['user'], 'aboy') - self.assertEqual(response.data['users'][2]['role'], 'editor') + self.assertEqual(response.data["users"][2]["user"], "aboy") + self.assertEqual(response.data["users"][2]["role"], "editor") def test_add_members_to_owner_role(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members', - 'get': 'retrieve', - 'put': 'members' - }) + view = OrganizationProfileViewSet.as_view( + {"post": "members", "get": "retrieve", "put": "members"} + ) - self.profile_data['username'] = "aboy" + self.profile_data["username"] = "aboy" aboy = self._create_user_profile().user - data = {'username': 'aboy', - 'role': 'owner'} + data = {"username": "aboy", "role": "owner"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', u'aboy'])) + self.assertEqual(set(response.data), set(["denoinc", "aboy"])) # getting profile - request = self.factory.get('/', **self.extra) - response = view(request, user='denoinc') + request = self.factory.get("/", **self.extra) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['users'][1]['user'], 'aboy') - self.assertEqual(response.data['users'][1]['role'], 'owner') + self.assertEqual(response.data["users"][1]["user"], "aboy") + self.assertEqual(response.data["users"][1]["role"], "owner") owner_team = get_or_create_organization_owners_team(self.organization) self.assertIn(aboy, owner_team.user_set.all()) # test user removed from owner team when role changed - data = {'username': 'aboy', 'role': 'editor'} + data = {"username": "aboy", "role": "editor"} request = self.factory.put( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) owner_team = get_or_create_organization_owners_team(self.organization) @@ -790,40 +761,37 @@ def test_add_members_to_owner_role(self): def test_org_members_added_to_projects(self): # create org self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members', - 'get': 'retrieve', - 'put': 'members' - }) + view = OrganizationProfileViewSet.as_view( + {"post": "members", "get": "retrieve", "put": "members"} + ) # create aboy - self.profile_data['username'] = "aboy" + self.profile_data["username"] = "aboy" aboy = self._create_user_profile().user - data = {'username': 'aboy', - 'role': 'owner'} + data = {"username": "aboy", "role": "owner"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) - response = view(request, user='denoinc') + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) # create a proj - project_data = { - 'owner': self.company_data['user'] - } + project_data = {"owner": self.company_data["user"]} self._project_create(project_data) self._publish_xls_form_to_project() # create alice - self.profile_data['username'] = "alice" + self.profile_data["username"] = "alice" alice = self._create_user_profile().user - alice_data = {'username': 'alice', - 'role': 'owner'} + alice_data = {"username": "alice", "role": "owner"} request = self.factory.post( - '/', data=json.dumps(alice_data), - content_type="application/json", **self.extra) - response = view(request, user='denoinc') + "/", + data=json.dumps(alice_data), + content_type="application/json", + **self.extra + ) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) # Assert that user added in org is added to teams in proj @@ -833,191 +801,188 @@ def test_org_members_added_to_projects(self): self.assertTrue(OwnerRole.user_has_role(alice, self.xform)) # Org admins are added to owners in project - projectView = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + projectView = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = projectView(request, pk=self.project.pk) - project_users = response.data.get('users') - users_in_users = [user['user'] for user in project_users] + project_users = response.data.get("users") + users_in_users = [user["user"] for user in project_users] - self.assertIn('bob', users_in_users) - self.assertIn('denoinc', users_in_users) - self.assertIn('aboy', users_in_users) - self.assertIn('alice', users_in_users) + self.assertIn("bob", users_in_users) + self.assertIn("denoinc", users_in_users) + self.assertIn("aboy", users_in_users) + self.assertIn("alice", users_in_users) def test_put_role_user_none_existent(self): self._org_create() - newname = 'i-do-no-exist' - view = OrganizationProfileViewSet.as_view({ - 'get': 'retrieve', - 'post': 'members', - 'put': 'members' - }) - - data = {'username': newname, 'role': 'editor'} + newname = "i-do-no-exist" + view = OrganizationProfileViewSet.as_view( + {"get": "retrieve", "post": "members", "put": "members"} + ) + + data = {"username": newname, "role": "editor"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 400) def test_update_org_name(self): self._org_create() # update name - data = {'name': "Dennis2"} - request = self.factory.patch('/', data=data, **self.extra) - response = self.view(request, user='denoinc') - self.assertEqual(response.data['name'], "Dennis2") + data = {"name": "Dennis2"} + request = self.factory.patch("/", data=data, **self.extra) + response = self.view(request, user="denoinc") + self.assertEqual(response.data["name"], "Dennis2") self.assertEqual(response.status_code, 200) # check in user profile endpoint - view_user = UserProfileViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view_user = UserProfileViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) - response = view_user(request, user='denoinc') - self.assertNotEqual(response.get('Cache-Control'), None) + response = view_user(request, user="denoinc") + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['name'], "Dennis2") + self.assertEqual(response.data["name"], "Dennis2") def test_org_always_has_admin_or_owner(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'put': 'members', - }) - data = {'username': self.user.username, 'role': 'editor'} + view = OrganizationProfileViewSet.as_view( + { + "put": "members", + } + ) + data = {"username": self.user.username, "role": "editor"} request = self.factory.put( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 400) - self.assertEquals(response.data, - { - u'non_field_errors': - [u'Organization cannot be without an owner'] - }) + self.assertEqual( + response.data, + {"non_field_errors": ["Organization cannot be without an owner"]}, + ) def test_owner_not_allowed_to_be_removed(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members', - 'delete': 'members', - 'get': 'retrieve', - }) + view = OrganizationProfileViewSet.as_view( + { + "post": "members", + "delete": "members", + "get": "retrieve", + } + ) - self.profile_data['username'] = "aboy" + self.profile_data["username"] = "aboy" aboy = self._create_user_profile().user - data = {'username': aboy.username, - 'role': 'owner'} + data = {"username": aboy.username, "role": "owner"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', - aboy.username])) + self.assertEqual(set(response.data), set(["denoinc", aboy.username])) - self.profile_data['username'] = "aboy2" + self.profile_data["username"] = "aboy2" aboy2 = self._create_user_profile().user - data = {'username': aboy2.username, - 'role': 'owner'} + data = {"username": aboy2.username, "role": "owner"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', - aboy.username, - aboy2.username])) + self.assertEqual( + set(response.data), set(["denoinc", aboy.username, aboy2.username]) + ) - data = {'username': aboy2.username} + data = {"username": aboy2.username} request = self.factory.delete( - '/', json.dumps(data), - content_type="application/json", **self.extra) + "/", json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - for user in [u'denoinc', aboy.username]: + for user in ["denoinc", aboy.username]: self.assertIn(user, response.data) # at this point we have bob and aboy as owners - data = {'username': aboy.username} + data = {"username": aboy.username} request = self.factory.delete( - '/', json.dumps(data), - content_type="application/json", **self.extra) + "/", json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) # at this point we only have bob as the owner - data = {'username': self.user.username} + data = {"username": self.user.username} request = self.factory.delete( - '/', json.dumps(data), - content_type="application/json", **self.extra) + "/", json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 400) - self.assertEquals(response.data, - { - u'non_field_errors': - [u'Organization cannot be without an owner'] - }) + self.assertEqual( + response.data, + {"non_field_errors": ["Organization cannot be without an owner"]}, + ) def test_orgs_delete(self): self._org_create() self.assertTrue(self.organization.user.is_active) - view = OrganizationProfileViewSet.as_view({ - 'delete': 'destroy' - }) + view = OrganizationProfileViewSet.as_view({"delete": "destroy"}) - request = self.factory.delete('/', **self.extra) - response = view(request, user='denoinc') + request = self.factory.delete("/", **self.extra) + response = view(request, user="denoinc") - self.assertEquals(204, response.status_code) + self.assertEqual(204, response.status_code) - self.assertEquals(0, OrganizationProfile.objects.filter( - user__username='denoinc').count()) - self.assertEquals(0, User.objects.filter(username='denoinc').count()) + self.assertEqual( + 0, OrganizationProfile.objects.filter(user__username="denoinc").count() + ) + self.assertEqual(0, User.objects.filter(username="denoinc").count()) def test_orgs_non_creator_delete(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'delete': 'members', - 'post': 'members' - }) + view = OrganizationProfileViewSet.as_view( + {"delete": "members", "post": "members"} + ) - self.profile_data['username'] = "alice" - self.profile_data['email'] = 'alice@localhost.com' + self.profile_data["username"] = "alice" + self.profile_data["email"] = "alice@localhost.com" self._create_user_profile() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} request = self.factory.post( - '/', data=json.dumps(alice_data), - content_type="application/json", **self.extra) + "/", + data=json.dumps(alice_data), + content_type="application/json", + **self.extra + ) - response = view(request, user='denoinc') - expected_results = [u'denoinc', u'alice'] + response = view(request, user="denoinc") + expected_results = ["denoinc", "alice"] self.assertEqual(status.HTTP_201_CREATED, response.status_code) self.assertEqual(expected_results, response.data) self._login_user_and_profile(extra_post_data=alice_data) - request = self.factory.delete('/', data=json.dumps(alice_data), - content_type="application/json", - **self.extra) - response = view(request, user='denoinc') - expected_results = [u'denoinc'] + request = self.factory.delete( + "/", + data=json.dumps(alice_data), + content_type="application/json", + **self.extra + ) + response = view(request, user="denoinc") + expected_results = ["denoinc"] self.assertEqual(expected_results, response.data) def test_creator_in_users(self): @@ -1026,19 +991,19 @@ def test_creator_in_users(self): in the value of the 'users' key within the response from /orgs """ self._org_create() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) expected_user = { - 'user': self.user.username, - 'role': 'owner', - 'first_name': 'Bob', - 'last_name': 'erama', - 'gravatar': self.user.profile.gravatar + "user": self.user.username, + "role": "owner", + "first_name": "Bob", + "last_name": "erama", + "gravatar": self.user.profile.gravatar, } - self.assertIn(expected_user, response.data[0]['users']) + self.assertIn(expected_user, response.data[0]["users"]) def test_creator_permissions(self): """ @@ -1046,9 +1011,9 @@ def test_creator_permissions(self): permissions """ self._org_create() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) orgs = OrganizationProfile.objects.filter(creator=self.user) @@ -1056,41 +1021,36 @@ def test_creator_permissions(self): org = orgs.first() self.assertTrue(OwnerRole.user_has_role(self.user, org)) - self.assertTrue( - OwnerRole.user_has_role(self.user, org.userprofile_ptr)) + self.assertTrue(OwnerRole.user_has_role(self.user, org.userprofile_ptr)) - members_view = OrganizationProfileViewSet.as_view({ - 'post': 'members', - 'delete': 'members' - }) + members_view = OrganizationProfileViewSet.as_view( + {"post": "members", "delete": "members"} + ) # New admins should also have the required permissions - self.profile_data['username'] = "dave" + self.profile_data["username"] = "dave" dave = self._create_user_profile().user - data = {'username': 'dave', - 'role': 'owner'} + data = {"username": "dave", "role": "owner"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) - response = members_view(request, user='denoinc') + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) + response = members_view(request, user="denoinc") self.assertEqual(response.status_code, 201) # Ensure user has role self.assertTrue(OwnerRole.user_has_role(dave, org)) - self.assertTrue( - OwnerRole.user_has_role(dave, org.userprofile_ptr)) + self.assertTrue(OwnerRole.user_has_role(dave, org.userprofile_ptr)) # Permissions should be removed when the user is removed from # organization - request = self.factory.delete('/', data=json.dumps(data), - content_type="application/json", - **self.extra) - response = members_view(request, user='denoinc') - expected_results = [u'denoinc'] + request = self.factory.delete( + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) + response = members_view(request, user="denoinc") + expected_results = ["denoinc"] self.assertEqual(expected_results, response.data) # Ensure permissions are removed self.assertFalse(OwnerRole.user_has_role(dave, org)) - self.assertFalse( - OwnerRole.user_has_role(dave, org.userprofile_ptr)) + self.assertFalse(OwnerRole.user_has_role(dave, org.userprofile_ptr)) diff --git a/onadata/apps/api/tests/viewsets/test_osm_viewset.py b/onadata/apps/api/tests/viewsets/test_osm_viewset.py index 1f5388bd5a..026bff9d4e 100644 --- a/onadata/apps/api/tests/viewsets/test_osm_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_osm_viewset.py @@ -130,7 +130,7 @@ def test_osm_csv_export(self): response = view(request, pk=self.xform.pk, format='csv') self.assertEqual(response.status_code, 200) - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) headers = dict(response.items()) self.assertEqual(headers['Content-Type'], 'application/csv') diff --git a/onadata/apps/api/tests/viewsets/test_project_viewset.py b/onadata/apps/api/tests/viewsets/test_project_viewset.py index f55ab99ed3..e56fbcd73b 100644 --- a/onadata/apps/api/tests/viewsets/test_project_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_project_viewset.py @@ -4,10 +4,10 @@ """ import json import os -from builtins import str -from future.utils import iteritems -from operator import itemgetter + from collections import OrderedDict +from six import iteritems +from operator import itemgetter from django.conf import settings from django.db.models import Q @@ -20,11 +20,14 @@ import requests from onadata.apps.api import tools -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import ( + TestAbstractViewSet, + get_mocked_response_for_file, +) from onadata.apps.api.tools import get_or_create_organization_owners_team -from onadata.apps.api.viewsets.organization_profile_viewset import \ - OrganizationProfileViewSet +from onadata.apps.api.viewsets.organization_profile_viewset import ( + OrganizationProfileViewSet, +) from onadata.apps.api.viewsets.project_viewset import ProjectViewSet from onadata.apps.api.viewsets.team_viewset import TeamViewSet from onadata.apps.api.viewsets.xform_viewset import XFormViewSet @@ -33,203 +36,224 @@ from onadata.libs import permissions as role from onadata.libs.models.share_project import ShareProject from onadata.libs.utils.cache_tools import PROJ_OWNER_CACHE, safe_key -from onadata.libs.permissions import (ROLES_ORDERED, DataEntryMinorRole, - DataEntryOnlyRole, DataEntryRole, - EditorMinorRole, EditorRole, ManagerRole, - OwnerRole, ReadOnlyRole, - ReadOnlyRoleNoDownload) -from onadata.libs.serializers.project_serializer import (BaseProjectSerializer, - ProjectSerializer) - -ROLES = [ReadOnlyRoleNoDownload, - ReadOnlyRole, - DataEntryRole, - EditorRole, - ManagerRole, - OwnerRole] - - -@urlmatch(netloc=r'(.*\.)?enketo\.ona\.io$') +from onadata.libs.permissions import ( + ROLES_ORDERED, + DataEntryMinorRole, + DataEntryOnlyRole, + DataEntryRole, + EditorMinorRole, + EditorRole, + ManagerRole, + OwnerRole, + ReadOnlyRole, + ReadOnlyRoleNoDownload, +) +from onadata.libs.serializers.project_serializer import ( + BaseProjectSerializer, + ProjectSerializer, +) + +ROLES = [ + ReadOnlyRoleNoDownload, + ReadOnlyRole, + DataEntryRole, + EditorRole, + ManagerRole, + OwnerRole, +] + + +# pylint: disable=unused-argument +@urlmatch(netloc=r"(.*\.)?enketo\.ona\.io$") def enketo_mock(url, request): + """Mock Enketo responses""" response = requests.Response() response.status_code = 201 - response._content = \ - '{\n "url": "https:\\/\\/dmfrm.enketo.org\\/webform",\n'\ - ' "code": "200"\n}' + setattr( + response, + "_content", + ('{\n "url": "https:\\/\\/dmfrm.enketo.org\\/webform",\n "code": "200"\n}'), + ) + return response def get_latest_tags(project): + """Return given project tags as a list.""" project.refresh_from_db() return [tag.name for tag in project.tags.all()] class TestProjectViewSet(TestAbstractViewSet): + """Test ProjectViewSet.""" def setUp(self): - super(TestProjectViewSet, self).setUp() - self.view = ProjectViewSet.as_view({ - 'get': 'list', - 'post': 'create' - }) + super().setUp() + self.view = ProjectViewSet.as_view({"get": "list", "post": "create"}) def tearDown(self): cache.clear() - super(TestProjectViewSet, self).tearDown() + super().tearDown() - @patch('onadata.apps.main.forms.urlopen') - def test_publish_xlsform_using_url_upload(self, mock_urlopen): + # pylint: disable=invalid-name + @patch("onadata.apps.main.forms.requests") + def test_publish_xlsform_using_url_upload(self, mock_requests): with HTTMock(enketo_mock): self._project_create() - view = ProjectViewSet.as_view({ - 'post': 'forms' - }) + view = ProjectViewSet.as_view({"post": "forms"}) pre_count = XForm.objects.count() project_id = self.project.pk - xls_url = 'https://ona.io/examples/forms/tutorial/form.xlsx' + xls_url = "https://ona.io/examples/forms/tutorial/form.xlsx" path = os.path.join( - settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", - "transportation", "transportation_different_id_string.xlsx") - - xls_file = open(path, 'rb') - mock_urlopen.return_value = xls_file - - post_data = {'xls_url': xls_url} - request = self.factory.post('/', data=post_data, **self.extra) - response = view(request, pk=project_id) - - mock_urlopen.assert_called_with(xls_url) - xls_file.close() - self.assertEqual(response.status_code, 201) - self.assertEqual(XForm.objects.count(), pre_count + 1) - self.assertEqual( - XFormVersion.objects.filter( - xform__pk=response.data.get('formid')).count(), 1) + settings.PROJECT_ROOT, + "apps", + "main", + "tests", + "fixtures", + "transportation", + "transportation_different_id_string.xlsx", + ) + + with open(path, "rb") as xls_file: + mock_response = get_mocked_response_for_file( + xls_file, "transportation_different_id_string.xlsx", 200 + ) + mock_requests.head.return_value = mock_response + mock_requests.get.return_value = mock_response + + post_data = {"xls_url": xls_url} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request, pk=project_id) + + mock_requests.get.assert_called_with(xls_url) + xls_file.close() + self.assertEqual(response.status_code, 201) + self.assertEqual(XForm.objects.count(), pre_count + 1) + self.assertEqual( + XFormVersion.objects.filter( + xform__pk=response.data.get("formid") + ).count(), + 1, + ) def test_projects_list(self): self._project_create() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.user = self.user response = self.view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - serializer = BaseProjectSerializer(self.project, - context={'request': request}) + serializer = BaseProjectSerializer(self.project, context={"request": request}) self.assertEqual(response.data, [serializer.data]) - self.assertIn('created_by', list(response.data[0])) + self.assertIn("created_by", list(response.data[0])) + # pylint: disable=invalid-name def test_project_list_returns_projects_for_active_users_only(self): + """Test project list returns projects of active users only.""" self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) alice_user = alice_profile.user - shared_project = Project(name='demo2', - shared=True, - metadata=json.dumps({'description': ''}), - created_by=alice_user, - organization=alice_user) + shared_project = Project( + name="demo2", + shared=True, + metadata=json.dumps({"description": ""}), + created_by=alice_user, + organization=alice_user, + ) shared_project.save() # share project with self.user - shareProject = ShareProject( - shared_project, self.user.username, 'manager') + shareProject = ShareProject(shared_project, self.user.username, "manager") shareProject.save() # ensure when alice_user isn't active we can NOT # see the project she shared alice_user.is_active = False alice_user.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.user = self.user response = self.view(request) self.assertEqual(len(response.data), 1) - self.assertNotEqual(response.data[0].get( - 'projectid'), shared_project.id) + self.assertNotEqual(response.data[0].get("projectid"), shared_project.id) # ensure when alice_user is active we can # see the project she shared alice_user.is_active = True alice_user.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.user = self.user response = self.view(request) self.assertEqual(len(response.data), 2) shared_project_in_response = False for project in response.data: - if project.get('projectid') == shared_project.id: + if project.get("projectid") == shared_project.id: shared_project_in_response = True break self.assertTrue(shared_project_in_response) + # pylint: disable=invalid-name def test_project_list_returns_users_own_project_is_shared_to(self): """ Ensure that the project list responses for project owners contains all the users the project has been shared too """ self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - share_project = ShareProject( - self.project, 'alice', 'manager' - ) + share_project = ShareProject(self.project, "alice", "manager") share_project.save() # Ensure alice is in the list of users # When an owner requests for the project data - req = self.factory.get('/', **self.extra) + req = self.factory.get("/", **self.extra) resp = self.view(req) self.assertEqual(resp.status_code, 200) - self.assertEqual(len(resp.data[0]['users']), 2) - shared_users = [user['user'] for user in resp.data[0]['users']] + self.assertEqual(len(resp.data[0]["users"]), 2) + shared_users = [user["user"] for user in resp.data[0]["users"]] self.assertIn(alice_profile.user.username, shared_users) # Ensure project managers can view all users the project was shared to - davis_data = {'username': 'davis', 'email': 'davis@localhost.com'} + davis_data = {"username": "davis", "email": "davis@localhost.com"} davis_profile = self._create_user_profile(davis_data) - dave_extras = { - 'HTTP_AUTHORIZATION': f'Token {davis_profile.user.auth_token}' - } + dave_extras = {"HTTP_AUTHORIZATION": f"Token {davis_profile.user.auth_token}"} share_project = ShareProject( - self.project, davis_profile.user.username, 'manager' + self.project, davis_profile.user.username, "manager" ) share_project.save() - req = self.factory.get('/', **dave_extras) + req = self.factory.get("/", **dave_extras) resp = self.view(req) self.assertEqual(resp.status_code, 200) - self.assertEqual(len(resp.data[0]['users']), 3) - shared_users = [user['user'] for user in resp.data[0]['users']] + self.assertEqual(len(resp.data[0]["users"]), 3) + shared_users = [user["user"] for user in resp.data[0]["users"]] self.assertIn(alice_profile.user.username, shared_users) self.assertIn(self.user.username, shared_users) def test_projects_get(self): self._project_create() - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - user_props = ['user', 'first_name', 'last_name', 'role', - 'is_org', 'metadata'] + user_props = ["user", "first_name", "last_name", "role", "is_org", "metadata"] user_props.sort() - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) # test serialized data - serializer = ProjectSerializer(self.project, - context={'request': request}) + serializer = ProjectSerializer(self.project, context={"request": request}) self.assertEqual(response.data, serializer.data) self.assertIsNotNone(self.project_data) self.assertEqual(response.data, self.project_data) - res_user_props = list(response.data['users'][0]) + res_user_props = list(response.data["users"][0]) res_user_props.sort() self.assertEqual(res_user_props, user_props) @@ -240,105 +264,110 @@ def test_project_get_deleted_form(self): self.xform.deleted_at = self.xform.date_created self.xform.save() - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertEqual(len(response.data.get('forms')), 0) + self.assertEqual(len(response.data.get("forms")), 0) self.assertEqual(response.status_code, 200) + # pylint: disable=invalid-name def test_none_empty_forms_and_dataview_properties_in_returned_json(self): self._publish_xls_form_to_project() self._create_dataview() - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertGreater(len(response.data.get('forms')), 0) - self.assertGreater( - len(response.data.get('data_views')), 0) - - form_obj_keys = list(response.data.get('forms')[0]) - data_view_obj_keys = list(response.data.get('data_views')[0]) - self.assertEqual(['date_created', - 'downloadable', - 'encrypted', - 'formid', - 'id_string', - 'is_merged_dataset', - 'last_submission_time', - 'last_updated_at', - 'name', - 'num_of_submissions', - 'published_by_formbuilder', - 'url'], - sorted(form_obj_keys)) - self.assertEqual(['columns', - 'dataviewid', - 'date_created', - 'date_modified', - 'instances_with_geopoints', - 'matches_parent', - 'name', - 'project', - 'query', - 'url', - 'xform'], - sorted(data_view_obj_keys)) + self.assertGreater(len(response.data.get("forms")), 0) + self.assertGreater(len(response.data.get("data_views")), 0) + + form_obj_keys = list(response.data.get("forms")[0]) + data_view_obj_keys = list(response.data.get("data_views")[0]) + self.assertEqual( + [ + "date_created", + "downloadable", + "encrypted", + "formid", + "id_string", + "is_merged_dataset", + "last_submission_time", + "last_updated_at", + "name", + "num_of_submissions", + "published_by_formbuilder", + "url", + ], + sorted(form_obj_keys), + ) + self.assertEqual( + [ + "columns", + "dataviewid", + "date_created", + "date_modified", + "instances_with_geopoints", + "matches_parent", + "name", + "project", + "query", + "url", + "xform", + ], + sorted(data_view_obj_keys), + ) self.assertEqual(response.status_code, 200) def test_projects_tags(self): self._project_create() - view = ProjectViewSet.as_view({ - 'get': 'labels', - 'post': 'labels', - 'delete': 'labels' - }) - list_view = ProjectViewSet.as_view({ - 'get': 'list', - }) + view = ProjectViewSet.as_view( + {"get": "labels", "post": "labels", "delete": "labels"} + ) + list_view = ProjectViewSet.as_view( + { + "get": "list", + } + ) project_id = self.project.pk # no tags - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=project_id) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.data, []) self.assertEqual(get_latest_tags(self.project), []) # add tag "hello" - request = self.factory.post('/', data={"tags": "hello"}, **self.extra) + request = self.factory.post("/", data={"tags": "hello"}, **self.extra) response = view(request, pk=project_id) self.assertEqual(response.status_code, 201) - self.assertEqual(response.data, [u'hello']) - self.assertEqual(get_latest_tags(self.project), [u'hello']) + self.assertEqual(response.data, ["hello"]) + self.assertEqual(get_latest_tags(self.project), ["hello"]) # check filter by tag - request = self.factory.get('/', data={"tags": "hello"}, **self.extra) + request = self.factory.get("/", data={"tags": "hello"}, **self.extra) self.project.refresh_from_db() request.user = self.user self.project_data = BaseProjectSerializer( - self.project, context={'request': request}).data + self.project, context={"request": request} + ).data response = list_view(request, pk=project_id) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data[0], self.project_data) - request = self.factory.get('/', data={"tags": "goodbye"}, **self.extra) + request = self.factory.get("/", data={"tags": "goodbye"}, **self.extra) response = list_view(request, pk=project_id) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) # remove tag "hello" - request = self.factory.delete('/', **self.extra) - response = view(request, pk=project_id, label='hello') + request = self.factory.delete("/", **self.extra) + response = view(request, pk=project_id, label="hello") self.assertEqual(response.status_code, 200) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) self.assertEqual(response.data, []) self.assertEqual(get_latest_tags(self.project), []) @@ -353,12 +382,12 @@ def test_projects_create(self): self.assertEqual(self.user, project.created_by) self.assertEqual(self.user, project.organization) - def test_project_create_other_account(self): # pylint: disable=C0103 + def test_project_create_other_account(self): # pylint: disable=invalid-name """ Test that a user cannot create a project in a different user account without the right permission. """ - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) bob = self.user self._login_user_and_profile(alice_data) @@ -368,17 +397,23 @@ def test_project_create_other_account(self): # pylint: disable=C0103 } # Alice should not be able to create the form in bobs account. - request = self.factory.post('/projects', data=data, **self.extra) + request = self.factory.post("/projects", data=data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, { - 'owner': [u'You do not have permission to create a project in ' - 'the organization {}.'.format(bob)]}) + self.assertEqual( + response.data, + { + "owner": [ + f"You do not have permission to create a project " + f"in the organization {bob}." + ] + }, + ) self.assertEqual(Project.objects.count(), 0) # Give Alice the permission to create a project in Bob's account. ManagerRole.add(alice_profile.user, bob.profile) - request = self.factory.post('/projects', data=data, **self.extra) + request = self.factory.post("/projects", data=data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 201) @@ -391,31 +426,31 @@ def test_project_create_other_account(self): # pylint: disable=C0103 # But under Bob's account self.assertEqual(bob, project.organization) + # pylint: disable=invalid-name def test_create_duplicate_project(self): """ Test creating a project with the same name """ # data to create project data = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False + "name": "demo", + "owner": f"http://testserver/api/v1/users/{self.user.username}", + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, } # current number of projects count = Project.objects.count() # create project using data - view = ProjectViewSet.as_view({ - 'post': 'create' - }) + view = ProjectViewSet.as_view({"post": "create"}) request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 201) after_count = Project.objects.count() @@ -423,28 +458,25 @@ def test_create_duplicate_project(self): # create another project using the same data request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data, { - u'non_field_errors': - [u'The fields name, owner must make a unique set.'] - } + response.data, + {"non_field_errors": ["The fields name, owner must make a unique set."]}, ) final_count = Project.objects.count() self.assertEqual(after_count, final_count) + # pylint: disable=invalid-name def test_projects_create_no_metadata(self): data = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'public': False + "name": "demo", + "owner": f"http://testserver/api/v1/users/{self.user.username}", + "public": False, } - self._project_create(project_data=data, - merge=False) + self._project_create(project_data=data, merge=False) self.assertIsNotNone(self.project) self.assertIsNotNone(self.project_data) @@ -455,9 +487,10 @@ def test_projects_create_no_metadata(self): self.assertEqual(self.user, project.created_by) self.assertEqual(self.user, project.organization) + # pylint: disable=invalid-name def test_projects_create_many_users(self): self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) self._project_create() projects = Project.objects.filter(created_by=self.user) @@ -467,237 +500,239 @@ def test_projects_create_many_users(self): self.assertEqual(self.user, project.created_by) self.assertEqual(self.user, project.organization) + # pylint: disable=invalid-name def test_publish_xls_form_to_project(self): self._publish_xls_form_to_project() - project_name = u'another project' - self._project_create({'name': project_name}) + project_name = "another project" + self._project_create({"name": project_name}) self._publish_xls_form_to_project() def test_num_datasets(self): self._publish_xls_form_to_project() self.project.refresh_from_db() - request = self.factory.post('/', data={}, **self.extra) + request = self.factory.post("/", data={}, **self.extra) request.user = self.user - self.project_data = ProjectSerializer( - self.project, context={'request': request}).data - self.assertEqual(self.project_data['num_datasets'], 1) + project_data = ProjectSerializer( + self.project, context={"request": request} + ).data + self.assertEqual(project_data["num_datasets"], 1) def test_last_submission_date(self): self._publish_xls_form_to_project() self._make_submissions() - request = self.factory.post('/', data={}, **self.extra) + request = self.factory.post("/", data={}, **self.extra) request.user = self.user self.project.refresh_from_db() - self.project_data = ProjectSerializer( - self.project, context={'request': request}).data - date_created = self.xform.instances.order_by( - '-date_created').values_list('date_created', flat=True)[0] - self.assertEqual(str(self.project_data['last_submission_date']), - str(date_created)) + project_data = ProjectSerializer( + self.project, context={"request": request} + ).data + date_created = self.xform.instances.order_by("-date_created").values_list( + "date_created", flat=True + )[0] + self.assertEqual(str(project_data["last_submission_date"]), str(date_created)) def test_view_xls_form(self): self._publish_xls_form_to_project() - view = ProjectViewSet.as_view({ - 'get': 'forms' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "forms"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) resultset = MetaData.objects.filter( - Q(object_id=self.xform.pk), Q(data_type='enketo_url') | - Q(data_type='enketo_preview_url') | - Q(data_type='enketo_single_submit_url')) - url = resultset.get(data_type='enketo_url') - preview_url = resultset.get(data_type='enketo_preview_url') - single_submit_url = resultset.get( - data_type='enketo_single_submit_url') - form_metadata = sorted([ + Q(object_id=self.xform.pk), + Q(data_type="enketo_url") + | Q(data_type="enketo_preview_url") + | Q(data_type="enketo_single_submit_url"), + ) + url = resultset.get(data_type="enketo_url") + preview_url = resultset.get(data_type="enketo_preview_url") + single_submit_url = resultset.get(data_type="enketo_single_submit_url") + form_metadata = sorted( + [ OrderedDict( [ - ('id', url.pk), - ('xform', self.xform.pk), - ('data_value', 'https://enketo.ona.io/::YY8M'), - ('data_type', 'enketo_url'), - ('data_file', None), - ('data_file_type', None), - ('media_url', None), - ('file_hash', None), - ('url', 'http://testserver/api/v1/metadata/%s' - % url.pk), - ('date_created', url.date_created)]), + ("id", url.pk), + ("xform", self.xform.pk), + ("data_value", "https://enketo.ona.io/::YY8M"), + ("data_type", "enketo_url"), + ("data_file", None), + ("data_file_type", None), + ("media_url", None), + ("file_hash", None), + ("url", f"http://testserver/api/v1/metadata/{url.pk}"), + ("date_created", url.date_created), + ] + ), OrderedDict( [ - ('id', preview_url.pk), - ('xform', self.xform.pk), - ('data_value', 'https://enketo.ona.io/preview/::YY8M'), - ('data_type', 'enketo_preview_url'), - ('data_file', None), - ('data_file_type', None), - ('media_url', None), - ('file_hash', None), - ('url', 'http://testserver/api/v1/metadata/%s' % - preview_url.pk), - ('date_created', preview_url.date_created)]), + ("id", preview_url.pk), + ("xform", self.xform.pk), + ("data_value", "https://enketo.ona.io/preview/::YY8M"), + ("data_type", "enketo_preview_url"), + ("data_file", None), + ("data_file_type", None), + ("media_url", None), + ("file_hash", None), + ( + "url", + f"http://testserver/api/v1/metadata/{preview_url.pk}", + ), + ("date_created", preview_url.date_created), + ] + ), OrderedDict( [ - ('id', single_submit_url.pk), - ('xform', self.xform.pk), - ('data_value', - 'http://enketo.ona.io/single/::XZqoZ94y'), - ('data_type', 'enketo_single_submit_url'), - ('data_file', None), - ('data_file_type', None), - ('media_url', None), - ('file_hash', None), - ('url', 'http://testserver/api/v1/metadata/%s' % - single_submit_url.pk), - ('date_created', single_submit_url.date_created)])], - key=itemgetter('id')) + ("id", single_submit_url.pk), + ("xform", self.xform.pk), + ("data_value", "http://enketo.ona.io/single/::XZqoZ94y"), + ("data_type", "enketo_single_submit_url"), + ("data_file", None), + ("data_file_type", None), + ("media_url", None), + ("file_hash", None), + ( + "url", + f"http://testserver/api/v1/metadata/{single_submit_url.pk}", + ), + ("date_created", single_submit_url.date_created), + ] + ), + ], + key=itemgetter("id"), + ) # test metadata content separately response_metadata = sorted( [dict(item) for item in response.data[0].pop("metadata")], - key=itemgetter('id')) + key=itemgetter("id"), + ) self.assertEqual(response_metadata, form_metadata) # remove metadata and date_modified - self.form_data.pop('metadata') - self.form_data.pop('date_modified') - self.form_data.pop('last_updated_at') - response.data[0].pop('date_modified') - response.data[0].pop('last_updated_at') - self.form_data.pop('has_id_string_changed') + self.form_data.pop("metadata") + self.form_data.pop("date_modified") + self.form_data.pop("last_updated_at") + response.data[0].pop("date_modified") + response.data[0].pop("last_updated_at") + self.form_data.pop("has_id_string_changed") self.assertDictEqual(dict(response.data[0]), dict(self.form_data)) def test_assign_form_to_project(self): - view = ProjectViewSet.as_view({ - 'post': 'forms', - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"post": "forms", "get": "retrieve"}) self._publish_xls_form_to_project() formid = self.xform.pk old_project = self.project - project_name = u'another project' - self._project_create({'name': project_name}) + project_name = "another project" + self._project_create({"name": project_name}) self.assertTrue(self.project.name == project_name) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=old_project.pk) self.assertEqual(response.status_code, 200) - self.assertIn('forms', list(response.data)) - old_project_form_count = len(response.data['forms']) - old_project_num_datasets = response.data['num_datasets'] + self.assertIn("forms", list(response.data)) + old_project_form_count = len(response.data["forms"]) + old_project_num_datasets = response.data["num_datasets"] project_id = self.project.pk - post_data = {'formid': formid} - request = self.factory.post('/', data=post_data, **self.extra) + post_data = {"formid": formid} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=project_id) self.assertEqual(response.status_code, 201) self.assertTrue(self.project.xform_set.filter(pk=self.xform.pk)) self.assertFalse(old_project.xform_set.filter(pk=self.xform.pk)) # check if form added appears in the project details - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertIn('forms', list(response.data)) - self.assertEqual(len(response.data['forms']), 1) - self.assertEqual(response.data['num_datasets'], 1) + self.assertIn("forms", list(response.data)) + self.assertEqual(len(response.data["forms"]), 1) + self.assertEqual(response.data["num_datasets"], 1) # Check if form transferred doesn't appear in the old project # details - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=old_project.pk) self.assertEqual(response.status_code, 200) - self.assertEqual( - len(response.data['forms']), old_project_form_count - 1) - self.assertEqual( - response.data['num_datasets'], old_project_num_datasets - 1) + self.assertEqual(len(response.data["forms"]), old_project_form_count - 1) + self.assertEqual(response.data["num_datasets"], old_project_num_datasets - 1) + # pylint: disable=invalid-name def test_project_manager_can_assign_form_to_project(self): - view = ProjectViewSet.as_view({ - 'post': 'forms', - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"post": "forms", "get": "retrieve"}) self._publish_xls_form_to_project() # alice user as manager to both projects - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - ShareProject(self.project, 'alice', 'manager').save() - self.assertTrue(ManagerRole.user_has_role(alice_profile.user, - self.project)) + ShareProject(self.project, "alice", "manager").save() + self.assertTrue(ManagerRole.user_has_role(alice_profile.user, self.project)) formid = self.xform.pk old_project = self.project - project_name = u'another project' - self._project_create({'name': project_name}) + project_name = "another project" + self._project_create({"name": project_name}) self.assertTrue(self.project.name == project_name) - ShareProject(self.project, 'alice', 'manager').save() - self.assertTrue(ManagerRole.user_has_role(alice_profile.user, - self.project)) + ShareProject(self.project, "alice", "manager").save() + self.assertTrue(ManagerRole.user_has_role(alice_profile.user, self.project)) self._login_user_and_profile(alice_data) project_id = self.project.pk - post_data = {'formid': formid} - request = self.factory.post('/', data=post_data, **self.extra) + post_data = {"formid": formid} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=project_id) self.assertEqual(response.status_code, 201) self.assertTrue(self.project.xform_set.filter(pk=self.xform.pk)) self.assertFalse(old_project.xform_set.filter(pk=self.xform.pk)) # check if form added appears in the project details - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertIn('forms', list(response.data)) - self.assertEqual(len(response.data['forms']), 1) + self.assertIn("forms", list(response.data)) + self.assertEqual(len(response.data["forms"]), 1) + # pylint: disable=invalid-name def test_project_manager_can_assign_form_to_project_no_perm(self): # user must have owner/manager permissions - view = ProjectViewSet.as_view({ - 'post': 'forms', - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"post": "forms", "get": "retrieve"}) self._publish_xls_form_to_project() # alice user is not manager to both projects - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - self.assertFalse(ManagerRole.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(ManagerRole.user_has_role(alice_profile.user, self.project)) formid = self.xform.pk - project_name = u'another project' - self._project_create({'name': project_name}) + project_name = "another project" + self._project_create({"name": project_name}) self.assertTrue(self.project.name == project_name) ManagerRole.add(alice_profile.user, self.project) - self.assertTrue(ManagerRole.user_has_role(alice_profile.user, - self.project)) + self.assertTrue(ManagerRole.user_has_role(alice_profile.user, self.project)) self._login_user_and_profile(alice_data) project_id = self.project.pk - post_data = {'formid': formid} - request = self.factory.post('/', data=post_data, **self.extra) + post_data = {"formid": formid} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=project_id) self.assertEqual(response.status_code, 403) + # pylint: disable=invalid-name def test_project_users_get_readonly_role_on_add_form(self): self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) ReadOnlyRole.add(alice_profile.user, self.project) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) self._publish_xls_form_to_project() - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.xform)) - self.assertFalse(OwnerRole.user_has_role(alice_profile.user, - self.xform)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) + self.assertFalse(OwnerRole.user_has_role(alice_profile.user, self.xform)) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + # pylint: disable=invalid-name,too-many-locals + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_reject_form_transfer_if_target_account_has_id_string_already( - self, mock_send_mail): + self, mock_send_mail + ): + """Test transfer form fails when a form with same id_string exists.""" # create bob's project and publish a form to it self._publish_xls_form_to_project() projectid = self.project.pk @@ -705,117 +740,121 @@ def test_reject_form_transfer_if_target_account_has_id_string_already( # create user alice alice_data = { - 'username': 'alice', - 'email': 'alice@localhost.com', - 'name': 'alice', - 'first_name': 'alice' + "username": "alice", + "email": "alice@localhost.com", + "name": "alice", + "first_name": "alice", } alice_profile = self._create_user_profile(alice_data) # share bob's project with alice - self.assertFalse( - ManagerRole.user_has_role(alice_profile.user, bobs_project)) - - data = {'username': 'alice', 'role': ManagerRole.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + self.assertFalse(ManagerRole.user_has_role(alice_profile.user, bobs_project)) + + data = { + "username": "alice", + "role": ManagerRole.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) self.assertTrue(mock_send_mail.called) - self.assertTrue( - ManagerRole.user_has_role(alice_profile.user, self.project)) - self.assertTrue( - ManagerRole.user_has_role(alice_profile.user, self.xform)) + self.assertTrue(ManagerRole.user_has_role(alice_profile.user, self.project)) + self.assertTrue(ManagerRole.user_has_role(alice_profile.user, self.xform)) # log in as alice self._login_user_and_profile(extra_post_data=alice_data) # publish a form to alice's project that shares an id_string with # form published by bob - publish_data = {'owner': 'http://testserver/api/v1/users/alice'} + publish_data = {"owner": "http://testserver/api/v1/users/alice"} self._publish_xls_form_to_project(publish_data=publish_data) alices_form = XForm.objects.filter( - user__username='alice', id_string='transportation_2011_07_25')[0] + user__username="alice", id_string="transportation_2011_07_25" + )[0] alices_project = alices_form.project bobs_form = XForm.objects.filter( - user__username='bob', id_string='transportation_2011_07_25')[0] + user__username="bob", id_string="transportation_2011_07_25" + )[0] formid = bobs_form.id # try transfering bob's form from bob's project to alice's project - view = ProjectViewSet.as_view({ - 'post': 'forms', - }) - post_data = {'formid': formid} - request = self.factory.post('/', data=post_data, **self.extra) + view = ProjectViewSet.as_view( + { + "post": "forms", + } + ) + post_data = {"formid": formid} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=alices_project.id) self.assertEqual(response.status_code, 400) - self.assertEquals( - response.data.get('detail'), - u'Form with the same id_string already exists in this account') + self.assertEqual( + response.data.get("detail"), + "Form with the same id_string already exists in this account", + ) # try transfering bob's form from to alice's other project with # no forms - self._project_create({'name': 'another project'}) + self._project_create({"name": "another project"}) new_project_id = self.project.id - view = ProjectViewSet.as_view({ - 'post': 'forms', - }) - post_data = {'formid': formid} - request = self.factory.post('/', data=post_data, **self.extra) + view = ProjectViewSet.as_view( + { + "post": "forms", + } + ) + post_data = {"formid": formid} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=new_project_id) self.assertEqual(response.status_code, 400) - self.assertEquals( - response.data.get('detail'), - u'Form with the same id_string already exists in this account') + self.assertEqual( + response.data.get("detail"), + "Form with the same id_string already exists in this account", + ) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') - def test_allow_form_transfer_if_org_is_owned_by_user( - self, mock_send_mail): + # pylint: disable=invalid-name + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") + def test_allow_form_transfer_if_org_is_owned_by_user(self, mock_send_mail): # create bob's project and publish a form to it self._publish_xls_form_to_project() bobs_project = self.project - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"get": "retrieve"}) # access bob's project initially to cache the forms list - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) view(request, pk=bobs_project.pk) # create an organization with a project self._org_create() - self._project_create({ - 'name': u'organization_project', - 'owner': 'http://testserver/api/v1/users/denoinc', - 'public': False - }) + self._project_create( + { + "name": "organization_project", + "owner": "http://testserver/api/v1/users/denoinc", + "public": False, + } + ) org_project = self.project self.assertNotEqual(bobs_project.id, org_project.id) # try transfering bob's form to an organization project he created - view = ProjectViewSet.as_view({ - 'post': 'forms', - 'get': 'retrieve' - }) - post_data = {'formid': self.xform.id} - request = self.factory.post('/', data=post_data, **self.extra) + view = ProjectViewSet.as_view({"post": "forms", "get": "retrieve"}) + post_data = {"formid": self.xform.id} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=self.project.id) self.assertEqual(response.status_code, 201) # test that cached forms of a source project are cleared. Bob had one # forms initially and now it's been moved to the org project. - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=bobs_project.pk) bobs_results = response.data - self.assertListEqual(bobs_results.get('forms'), []) + self.assertListEqual(bobs_results.get("forms"), []) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + # pylint: disable=invalid-name + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_handle_integrity_error_on_form_transfer(self, mock_send_mail): # create bob's project and publish a form to it self._publish_xls_form_to_project() @@ -823,60 +862,67 @@ def test_handle_integrity_error_on_form_transfer(self, mock_send_mail): # create an organization with a project self._org_create() - self._project_create({ - 'name': u'organization_project', - 'owner': 'http://testserver/api/v1/users/denoinc', - 'public': False - }) + self._project_create( + { + "name": "organization_project", + "owner": "http://testserver/api/v1/users/denoinc", + "public": False, + } + ) # publish form to organization project self._publish_xls_form_to_project() # try transfering bob's form to an organization project he created - view = ProjectViewSet.as_view({ - 'post': 'forms', - }) - post_data = {'formid': xform.id} - request = self.factory.post('/', data=post_data, **self.extra) + view = ProjectViewSet.as_view( + { + "post": "forms", + } + ) + post_data = {"formid": xform.id} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=self.project.id) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data.get('detail'), - u'Form with the same id_string already exists in this account') + response.data.get("detail"), + "Form with the same id_string already exists in this account", + ) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') - def test_form_transfer_when_org_creator_creates_project( - self, mock_send_mail): + # pylint: disable=invalid-name + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") + def test_form_transfer_when_org_creator_creates_project(self, mock_send_mail): projects_count = Project.objects.count() xform_count = XForm.objects.count() user_bob = self.user # create user alice with a project - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) self._login_user_and_profile(alice_data) - self._project_create({ - 'name': u'alice\'s project', - 'owner': ('http://testserver/api/v1/users/%s' - % alice_profile.user.username), - 'public': False, - }, merge=False) + self._project_create( + { + "name": "alice's project", + "owner": ( + f"http://testserver/api/v1/users/{alice_profile.user.username}" + ), + "public": False, + }, + merge=False, + ) self.assertEqual(self.project.created_by, alice_profile.user) alice_project = self.project # create org owned by bob then make alice admin self._login_user_and_profile( - {'username': user_bob.username, 'email': user_bob.email}) + {"username": user_bob.username, "email": user_bob.email} + ) self._org_create() self.assertEqual(self.organization.created_by, user_bob) - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) - data = {'username': alice_profile.user.username, - 'role': OwnerRole.name} + view = OrganizationProfileViewSet.as_view({"post": "members"}) + data = {"username": alice_profile.user.username, "role": OwnerRole.name} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, user=self.organization.user.username) self.assertEqual(response.status_code, 201) @@ -884,11 +930,13 @@ def test_form_transfer_when_org_creator_creates_project( self.assertIn(alice_profile.user, owners_team.user_set.all()) # let bob create a project in org - self._project_create({ - 'name': u'organization_project', - 'owner': 'http://testserver/api/v1/users/denoinc', - 'public': False, - }) + self._project_create( + { + "name": "organization_project", + "owner": "http://testserver/api/v1/users/denoinc", + "public": False, + } + ) self.assertEqual(self.project.created_by, user_bob) org_project = self.project self.assertEqual(Project.objects.count(), projects_count + 2) @@ -897,32 +945,34 @@ def test_form_transfer_when_org_creator_creates_project( self._login_user_and_profile(alice_data) self.project = alice_project data = { - 'owner': ('http://testserver/api/v1/users/%s' - % alice_profile.user.username), - 'public': True, - 'public_data': True, - 'description': u'transportation_2011_07_25', - 'downloadable': True, - 'allows_sms': False, - 'encrypted': False, - 'sms_id_string': u'transportation_2011_07_25', - 'id_string': u'transportation_2011_07_25', - 'title': u'transportation_2011_07_25', - 'bamboo_dataset': u'' + "owner": ("http://testserver/api/v1/users/{alice_profile.user.username}"), + "public": True, + "public_data": True, + "description": "transportation_2011_07_25", + "downloadable": True, + "allows_sms": False, + "encrypted": False, + "sms_id_string": "transportation_2011_07_25", + "id_string": "transportation_2011_07_25", + "title": "transportation_2011_07_25", + "bamboo_dataset": "", } self._publish_xls_form_to_project(publish_data=data, merge=False) self.assertEqual(self.xform.created_by, alice_profile.user) self.assertEqual(XForm.objects.count(), xform_count + 1) # let alice transfer the form to the organization project - view = ProjectViewSet.as_view({ - 'post': 'forms', - }) - post_data = {'formid': self.xform.id} - request = self.factory.post('/', data=post_data, **self.extra) + view = ProjectViewSet.as_view( + { + "post": "forms", + } + ) + post_data = {"formid": self.xform.id} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=org_project.id) self.assertEqual(response.status_code, 201) + # pylint: disable=invalid-name def test_project_transfer_upgrades_permissions(self): """ Test that existing project permissions are updated when necessary @@ -931,34 +981,31 @@ def test_project_transfer_upgrades_permissions(self): bob = self.user # create user alice with a project - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) self._login_user_and_profile(alice_data) alice_url = f'http://testserver/api/v1/users/{alice_data["username"]}' - self._project_create({ - 'name': 'test project', - 'owner': alice_url, - 'public': False, - }, merge=False) + self._project_create( + { + "name": "test project", + "owner": alice_url, + "public": False, + }, + merge=False, + ) self.assertEqual(self.project.created_by, alice_profile.user) alice_project = self.project # create org owned by bob then make alice admin - self._login_user_and_profile( - {'username': bob.username, 'email': bob.email}) + self._login_user_and_profile({"username": bob.username, "email": bob.email}) self._org_create() self.assertEqual(self.organization.created_by, bob) - org_url = ( - 'http://testserver/api/v1/users/' - f'{self.organization.user.username}') - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) - data = {'username': alice_profile.user.username, - 'role': OwnerRole.name} + org_url = f"http://testserver/api/v1/users/{self.organization.user.username}" + view = OrganizationProfileViewSet.as_view({"post": "members"}) + data = {"username": alice_profile.user.username, "role": OwnerRole.name} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, user=self.organization.user.username) self.assertEqual(response.status_code, 201) @@ -966,100 +1013,95 @@ def test_project_transfer_upgrades_permissions(self): self.assertIn(alice_profile.user, owners_team.user_set.all()) # Share project to bob as editor - data = {'username': bob.username, 'role': EditorRole.name} - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + data = {"username": bob.username, "role": EditorRole.name} + view = ProjectViewSet.as_view({"post": "share"}) alice_auth_token = Token.objects.get(user=alice_profile.user).key - auth_credentials = { - 'HTTP_AUTHORIZATION': f'Token {alice_auth_token}' - } - request = self.factory.post('/', data=data, **auth_credentials) + auth_credentials = {"HTTP_AUTHORIZATION": f"Token {alice_auth_token}"} + request = self.factory.post("/", data=data, **auth_credentials) response = view(request, pk=alice_project.pk) self.assertEqual(response.status_code, 204) # Transfer project to Bobs Organization - data = {'owner': org_url, 'name': alice_project.name} - view = ProjectViewSet.as_view({ - 'patch': 'partial_update' - }) - request = self.factory.patch('/', data=data, **auth_credentials) + data = {"owner": org_url, "name": alice_project.name} + view = ProjectViewSet.as_view({"patch": "partial_update"}) + request = self.factory.patch("/", data=data, **auth_credentials) response = view(request, pk=alice_project.pk) self.assertEqual(response.status_code, 200) # Ensure all Admins have admin privileges to the project # once transferred - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=alice_project.pk) self.assertEqual(response.status_code, 200) - project_users = response.data['users'] + project_users = response.data["users"] org_owners = get_or_create_organization_owners_team( - self.organization).user_set.all() + self.organization + ).user_set.all() for user in project_users: - owner = org_owners.filter(username=user['user']).first() + owner = org_owners.filter(username=user["user"]).first() if owner: - self.assertEqual(user['role'], OwnerRole.name) + self.assertEqual(user["role"], OwnerRole.name) + # pylint: disable=invalid-name @override_settings(ALLOW_PUBLIC_DATASETS=False) def test_disallow_public_project_creation(self): """ Test that an error is raised when a user tries to create a public project when public projects are disabled. """ - view = ProjectViewSet.as_view({ - 'post': 'create' - }) + view = ProjectViewSet.as_view({"post": "create"}) data = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'public': True + "name": "demo", + "owner": f"http://testserver/api/v1/users/{self.user.username}", + "public": True, } - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data['public'][0], - "Public projects are currently disabled.") + response.data["public"][0], "Public projects are currently disabled." + ) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + # pylint: disable=invalid-name + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_form_transfer_when_org_admin_not_creator_creates_project( - self, mock_send_mail): + self, mock_send_mail + ): projects_count = Project.objects.count() xform_count = XForm.objects.count() user_bob = self.user # create user alice with a project - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) self._login_user_and_profile(alice_data) - self._project_create({ - 'name': u'alice\'s project', - 'owner': ('http://testserver/api/v1/users/%s' - % alice_profile.user.username), - 'public': False, - }, merge=False) + self._project_create( + { + "name": "alice's project", + "owner": ( + f"http://testserver/api/v1/users/{alice_profile.user.username}" + ), + "public": False, + }, + merge=False, + ) self.assertEqual(self.project.created_by, alice_profile.user) alice_project = self.project # create org owned by bob then make alice admin self._login_user_and_profile( - {'username': user_bob.username, 'email': user_bob.email}) + {"username": user_bob.username, "email": user_bob.email} + ) self._org_create() self.assertEqual(self.organization.created_by, user_bob) - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) - data = {'username': alice_profile.user.username, - 'role': OwnerRole.name} + view = OrganizationProfileViewSet.as_view({"post": "members"}) + data = {"username": alice_profile.user.username, "role": OwnerRole.name} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, user=self.organization.user.username) self.assertEqual(response.status_code, 201) @@ -1068,11 +1110,13 @@ def test_form_transfer_when_org_admin_not_creator_creates_project( # let alice create a project in org self._login_user_and_profile(alice_data) - self._project_create({ - 'name': u'organization_project', - 'owner': 'http://testserver/api/v1/users/denoinc', - 'public': False, - }) + self._project_create( + { + "name": "organization_project", + "owner": "http://testserver/api/v1/users/denoinc", + "public": False, + } + ) self.assertEqual(self.project.created_by, alice_profile.user) org_project = self.project self.assertEqual(Project.objects.count(), projects_count + 2) @@ -1080,210 +1124,216 @@ def test_form_transfer_when_org_admin_not_creator_creates_project( # let alice create a form in her personal project self.project = alice_project data = { - 'owner': ('http://testserver/api/v1/users/%s' - % alice_profile.user.username), - 'public': True, - 'public_data': True, - 'description': u'transportation_2011_07_25', - 'downloadable': True, - 'allows_sms': False, - 'encrypted': False, - 'sms_id_string': u'transportation_2011_07_25', - 'id_string': u'transportation_2011_07_25', - 'title': u'transportation_2011_07_25', - 'bamboo_dataset': u'' + "owner": ("http://testserver/api/v1/users/{alice_profile.user.username}"), + "public": True, + "public_data": True, + "description": "transportation_2011_07_25", + "downloadable": True, + "allows_sms": False, + "encrypted": False, + "sms_id_string": "transportation_2011_07_25", + "id_string": "transportation_2011_07_25", + "title": "transportation_2011_07_25", + "bamboo_dataset": "", } self._publish_xls_form_to_project(publish_data=data, merge=False) self.assertEqual(self.xform.created_by, alice_profile.user) self.assertEqual(XForm.objects.count(), xform_count + 1) # let alice transfer the form to the organization project - view = ProjectViewSet.as_view({ - 'post': 'forms', - }) - post_data = {'formid': self.xform.id} - request = self.factory.post('/', data=post_data, **self.extra) + view = ProjectViewSet.as_view( + { + "post": "forms", + } + ) + post_data = {"formid": self.xform.id} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=org_project.id) self.assertEqual(response.status_code, 201) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + # pylint: disable=invalid-name + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_project_share_endpoint(self, mock_send_mail): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) projectid = self.project.pk for role_class in ROLES: - self.assertFalse(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(role_class.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': role_class.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) + data = { + "username": "alice", + "role": role_class.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) self.assertTrue(mock_send_mail.called) - self.assertTrue(role_class.user_has_role(alice_profile.user, - self.project)) - self.assertTrue(role_class.user_has_role(alice_profile.user, - self.xform)) + self.assertTrue(role_class.user_has_role(alice_profile.user, self.project)) + self.assertTrue(role_class.user_has_role(alice_profile.user, self.xform)) # Reset the mock called value to False mock_send_mail.called = False - data = {'username': 'alice', 'role': ''} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "alice", "role": ""} + request = self.factory.post("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) self.assertFalse(mock_send_mail.called) - role_class._remove_obj_permissions(alice_profile.user, - self.project) + # pylint: disable=protected-access + role_class._remove_obj_permissions(alice_profile.user, self.project) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + # pylint: disable=invalid-name + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_project_share_endpoint_form_published_later(self, mock_send_mail): # create project self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) projectid = self.project.pk for role_class in ROLES: - self.assertFalse(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(role_class.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': role_class.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) + data = { + "username": "alice", + "role": role_class.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) self.assertTrue(mock_send_mail.called) - self.assertTrue(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertTrue(role_class.user_has_role(alice_profile.user, self.project)) # publish form after project sharing self._publish_xls_form_to_project() - self.assertTrue(role_class.user_has_role(alice_profile.user, - self.xform)) + self.assertTrue(role_class.user_has_role(alice_profile.user, self.xform)) # Reset the mock called value to False mock_send_mail.called = False - data = {'username': 'alice', 'role': ''} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "alice", "role": ""} + request = self.factory.post("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) self.assertFalse(mock_send_mail.called) - role_class._remove_obj_permissions(alice_profile.user, - self.project) + # pylint: disable=protected-access + role_class._remove_obj_permissions(alice_profile.user, self.project) self.xform.delete() def test_project_share_remove_user(self): self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) projectid = self.project.pk role_class = ReadOnlyRole - data = {'username': 'alice', 'role': role_class.name} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "alice", "role": role_class.name} + request = self.factory.post("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - self.assertTrue(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertTrue(role_class.user_has_role(alice_profile.user, self.project)) - data['remove'] = True - request = self.factory.post('/', data=data, **self.extra) + data["remove"] = True + request = self.factory.post("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - self.assertFalse(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(role_class.user_has_role(alice_profile.user, self.project)) + # pylint: disable=too-many-statements def test_project_filter_by_owner(self): """ Test projects endpoint filter by owner. """ self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com', - 'first_name': 'Alice', 'last_name': 'Alice'} + alice_data = { + "username": "alice", + "email": "alice@localhost.com", + "first_name": "Alice", + "last_name": "Alice", + } self._login_user_and_profile(alice_data) - ShareProject(self.project, self.user.username, 'readonly').save() + ShareProject(self.project, self.user.username, "readonly").save() - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', {'owner': 'bob'}, **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", {"owner": "bob"}, **self.extra) response = view(request, pk=self.project.pk) request.user = self.user self.project.refresh_from_db() bobs_project_data = BaseProjectSerializer( - self.project, context={'request': request}).data + self.project, context={"request": request} + ).data - self._project_create({'name': 'another project'}) + self._project_create({"name": "another project"}) # both bob's and alice's projects - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) - request = self.factory.get('/', {'owner': 'alice'}, **self.extra) + request = self.factory.get("/", {"owner": "alice"}, **self.extra) request.user = self.user alice_project_data = BaseProjectSerializer( - self.project, context={'request': request}).data - result = [{'owner': p.get('owner'), - 'projectid': p.get('projectid')} for p in response.data] - bob_data = {'owner': 'http://testserver/api/v1/users/bob', - 'projectid': bobs_project_data.get('projectid')} - alice_data = {'owner': 'http://testserver/api/v1/users/alice', - 'projectid': alice_project_data.get('projectid')} + self.project, context={"request": request} + ).data + result = [ + {"owner": p.get("owner"), "projectid": p.get("projectid")} + for p in response.data + ] + bob_data = { + "owner": "http://testserver/api/v1/users/bob", + "projectid": bobs_project_data.get("projectid"), + } + alice_data = { + "owner": "http://testserver/api/v1/users/alice", + "projectid": alice_project_data.get("projectid"), + } self.assertIn(bob_data, result) self.assertIn(alice_data, result) # only bob's project - request = self.factory.get('/', {'owner': 'bob'}, **self.extra) + request = self.factory.get("/", {"owner": "bob"}, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertIn(bobs_project_data, response.data) self.assertNotIn(alice_project_data, response.data) # only alice's project - request = self.factory.get('/', {'owner': 'alice'}, **self.extra) + request = self.factory.get("/", {"owner": "alice"}, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertNotIn(bobs_project_data, response.data) self.assertIn(alice_project_data, response.data) # none existent user - request = self.factory.get('/', {'owner': 'noone'}, **self.extra) + request = self.factory.get("/", {"owner": "noone"}, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) # authenticated user can view public project - joe_data = {'username': 'joe', 'email': 'joe@localhost.com'} + joe_data = {"username": "joe", "email": "joe@localhost.com"} self._login_user_and_profile(joe_data) # should not show private projects when filtered by owner - request = self.factory.get('/', {'owner': 'alice'}, **self.extra) + request = self.factory.get("/", {"owner": "alice"}, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertNotIn(bobs_project_data, response.data) @@ -1294,114 +1344,112 @@ def test_project_filter_by_owner(self): self.project.save() request.user = self.user alice_project_data = BaseProjectSerializer( - self.project, context={'request': request}).data + self.project, context={"request": request} + ).data - request = self.factory.get('/', {'owner': 'alice'}, **self.extra) + request = self.factory.get("/", {"owner": "alice"}, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertIn(alice_project_data, response.data) # should show deleted project public project when filtered by owner self.project.soft_delete() - request = self.factory.get('/', {'owner': 'alice'}, **self.extra) + request = self.factory.get("/", {"owner": "alice"}, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual([], response.data) def test_project_partial_updates(self): self._project_create() - view = ProjectViewSet.as_view({ - 'patch': 'partial_update' - }) + view = ProjectViewSet.as_view({"patch": "partial_update"}) projectid = self.project.pk - metadata = '{"description": "Lorem ipsum",' \ - '"location": "Nakuru, Kenya",' \ - '"category": "water"' \ - '}' + metadata = ( + '{"description": "Lorem ipsum",' + '"location": "Nakuru, Kenya",' + '"category": "water"' + "}" + ) json_metadata = json.loads(metadata) - data = {'metadata': metadata} - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": metadata} + request = self.factory.patch("/", data=data, **self.extra) response = view(request, pk=projectid) project = Project.objects.get(pk=projectid) self.assertEqual(response.status_code, 200) self.assertEqual(project.metadata, json_metadata) + # pylint: disable=invalid-name def test_cache_updated_on_project_update(self): - view = ProjectViewSet.as_view({ - 'get': 'retrieve', - 'patch': 'partial_update' - }) + view = ProjectViewSet.as_view({"get": "retrieve", "patch": "partial_update"}) self._project_create() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) self.assertEqual(False, response.data.get("public")) - cached_project = cache.get(f'{PROJ_OWNER_CACHE}{self.project.pk}') + cached_project = cache.get(f"{PROJ_OWNER_CACHE}{self.project.pk}") self.assertEqual(cached_project, response.data) projectid = self.project.pk - data = {'public': True} - request = self.factory.patch('/', data=data, **self.extra) + data = {"public": True} + request = self.factory.patch("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 200) self.assertEqual(True, response.data.get("public")) - cached_project = cache.get(f'{PROJ_OWNER_CACHE}{self.project.pk}') + cached_project = cache.get(f"{PROJ_OWNER_CACHE}{self.project.pk}") self.assertEqual(cached_project, response.data) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) self.assertEqual(True, response.data.get("public")) - cached_project = cache.get(f'{PROJ_OWNER_CACHE}{self.project.pk}') + cached_project = cache.get(f"{PROJ_OWNER_CACHE}{self.project.pk}") self.assertEqual(cached_project, response.data) def test_project_put_updates(self): self._project_create() - view = ProjectViewSet.as_view({ - 'put': 'update' - }) + view = ProjectViewSet.as_view({"put": "update"}) projectid = self.project.pk data = { - 'name': u'updated name', - 'owner': 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'description', - 'location': 'Nairobi, Kenya', - 'category': 'health'} + "name": "updated name", + "owner": f"http://testserver/api/v1/users/{self.user.username}", + "metadata": { + "description": "description", + "location": "Nairobi, Kenya", + "category": "health", + }, } - data.update({'metadata': json.dumps(data.get('metadata'))}) - request = self.factory.put('/', data=data, **self.extra) + data.update({"metadata": json.dumps(data.get("metadata"))}) + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=projectid) - data.update({'metadata': json.loads(data.get('metadata'))}) + data.update({"metadata": json.loads(data.get("metadata"))}) self.assertDictContainsSubset(data, response.data) + # pylint: disable=invalid-name def test_project_partial_updates_to_existing_metadata(self): self._project_create() - view = ProjectViewSet.as_view({ - 'patch': 'partial_update' - }) + view = ProjectViewSet.as_view({"patch": "partial_update"}) projectid = self.project.pk metadata = '{"description": "Changed description"}' json_metadata = json.loads(metadata) - data = {'metadata': metadata} - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": metadata} + request = self.factory.patch("/", data=data, **self.extra) response = view(request, pk=projectid) project = Project.objects.get(pk=projectid) json_metadata.update(project.metadata) self.assertEqual(response.status_code, 200) self.assertEqual(project.metadata, json_metadata) + # pylint: disable=invalid-name def test_project_update_shared_cascades_to_xforms(self): self._publish_xls_form_to_project() - view = ProjectViewSet.as_view({ - 'patch': 'partial_update' - }) + view = ProjectViewSet.as_view({"patch": "partial_update"}) projectid = self.project.pk - data = {'public': 'true'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"public": "true"} + request = self.factory.patch("/", data=data, **self.extra) response = view(request, pk=projectid) - xforms_status = XForm.objects.filter(project__pk=projectid)\ - .values_list('shared', flat=True) + xforms_status = XForm.objects.filter(project__pk=projectid).values_list( + "shared", flat=True + ) self.assertTrue(xforms_status[0]) self.assertEqual(response.status_code, 200) @@ -1409,54 +1457,46 @@ def test_project_add_star(self): self._project_create() self.assertEqual(len(self.project.user_stars.all()), 0) - view = ProjectViewSet.as_view({ - 'post': 'star' - }) - request = self.factory.post('/', **self.extra) + view = ProjectViewSet.as_view({"post": "star"}) + request = self.factory.post("/", **self.extra) response = view(request, pk=self.project.pk) self.project.refresh_from_db() self.assertEqual(response.status_code, 204) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) self.assertEqual(len(self.project.user_stars.all()), 1) self.assertEqual(self.project.user_stars.all()[0], self.user) + # pylint: disable=invalid-name def test_create_project_invalid_metadata(self): """ Make sure that invalid metadata values are outright rejected Test fix for: https://github.com/onaio/onadata/issues/977 """ - view = ProjectViewSet.as_view({ - 'post': 'create' - }) + view = ProjectViewSet.as_view({"post": "create"}) data = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': "null", - 'public': False + "name": "demo", + "owner": f"http://testserver/api/v1/users/{self.user.username}", + "metadata": "null", + "public": False, } request = self.factory.post( - '/', - data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 400) def test_project_delete_star(self): self._project_create() - view = ProjectViewSet.as_view({ - 'delete': 'star', - 'post': 'star' - }) - request = self.factory.post('/', **self.extra) + view = ProjectViewSet.as_view({"delete": "star", "post": "star"}) + request = self.factory.post("/", **self.extra) response = view(request, pk=self.project.pk) self.project.refresh_from_db() self.assertEqual(len(self.project.user_stars.all()), 1) self.assertEqual(self.project.user_stars.all()[0], self.user) - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = view(request, pk=self.project.pk) self.project.refresh_from_db() @@ -1467,70 +1507,65 @@ def test_project_get_starred_by(self): self._project_create() # add star as bob - view = ProjectViewSet.as_view({ - 'get': 'star', - 'post': 'star' - }) - request = self.factory.post('/', **self.extra) + view = ProjectViewSet.as_view({"get": "star", "post": "star"}) + request = self.factory.post("/", **self.extra) response = view(request, pk=self.project.pk) # ensure email not shared user_profile_data = self.user_profile_data() - del user_profile_data['email'] - del user_profile_data['metadata'] + del user_profile_data["email"] + del user_profile_data["metadata"] - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) # add star as alice - request = self.factory.post('/', **self.extra) + request = self.factory.post("/", **self.extra) response = view(request, pk=self.project.pk) # get star users as alice - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) - alice_profile, bob_profile = sorted(response.data, - key=itemgetter('username')) - self.assertEquals(sorted(bob_profile.items()), - sorted(user_profile_data.items())) - self.assertEqual(alice_profile['username'], 'alice') + alice_profile, bob_profile = sorted(response.data, key=itemgetter("username")) + self.assertEqual(sorted(bob_profile.items()), sorted(user_profile_data.items())) + self.assertEqual(alice_profile["username"], "alice") def test_user_can_view_public_projects(self): - public_project = Project(name='demo', - shared=True, - metadata=json.dumps({'description': ''}), - created_by=self.user, - organization=self.user) + public_project = Project( + name="demo", + shared=True, + metadata=json.dumps({"description": ""}), + created_by=self.user, + organization=self.user, + ) public_project.save() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=public_project.pk) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['public'], True) - self.assertEqual(response.data['projectid'], public_project.pk) - self.assertEqual(response.data['name'], 'demo') + self.assertEqual(response.data["public"], True) + self.assertEqual(response.data["projectid"], public_project.pk) + self.assertEqual(response.data["name"], "demo") def test_projects_same_name_diff_case(self): data1 = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False + "name": "demo", + "owner": f"http://testserver/api/v1/users/{self.user.username}", + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, } - self._project_create(project_data=data1, - merge=False) + self._project_create(project_data=data1, merge=False) self.assertIsNotNone(self.project) self.assertIsNotNone(self.project_data) @@ -1538,25 +1573,24 @@ def test_projects_same_name_diff_case(self): self.assertEqual(len(projects), 1) data2 = { - 'name': u'DEMO', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False + "name": "DEMO", + "owner": f"http://testserver/api/v1/users/{self.user.username}", + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, } - view = ProjectViewSet.as_view({ - 'post': 'create' - }) + view = ProjectViewSet.as_view({"post": "create"}) request = self.factory.post( - '/', data=json.dumps(data2), - content_type="application/json", **self.extra) + "/", data=json.dumps(data2), content_type="application/json", **self.extra + ) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) projects = Project.objects.all() self.assertEqual(len(projects), 1) @@ -1565,41 +1599,44 @@ def test_projects_same_name_diff_case(self): self.assertEqual(self.user, project.organization) def test_projects_get_exception(self): - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) # does not exists response = view(request, pk=11111) self.assertEqual(response.status_code, 404) - self.assertEqual(response.data, {u'detail': u'Not found.'}) + self.assertEqual(response.data, {"detail": "Not found."}) # invalid id - response = view(request, pk='1w') + response = view(request, pk="1w") self.assertEqual(response.status_code, 400) - error_msg = ("Invalid value for project_id. It must be a " - "positive integer.") - self.assertEqual(str(response.data['detail']), error_msg) + error_msg = "Invalid value for project_id. It must be a positive integer." + self.assertEqual(str(response.data["detail"]), error_msg) def test_publish_to_public_project(self): - public_project = Project(name='demo', - shared=True, - metadata=json.dumps({'description': ''}), - created_by=self.user, - organization=self.user) + public_project = Project( + name="demo", + shared=True, + metadata=json.dumps({"description": ""}), + created_by=self.user, + organization=self.user, + ) public_project.save() self.project = public_project self._publish_xls_form_to_project(public=True) - self.assertEquals(self.xform.shared, True) - self.assertEquals(self.xform.shared_data, True) + self.assertEqual(self.xform.shared, True) + self.assertEqual(self.xform.shared_data, True) def test_public_form_private_project(self): - self.project = Project(name='demo', shared=False, - metadata=json.dumps({'description': ''}), - created_by=self.user, organization=self.user) + self.project = Project( + name="demo", + shared=False, + metadata=json.dumps({"description": ""}), + created_by=self.user, + organization=self.user, + ) self.project.save() self._publish_xls_form_to_project() @@ -1642,108 +1679,97 @@ def test_public_form_private_project(self): self.assertFalse(self.project.shared) def test_publish_to_public_project_public_form(self): - public_project = Project(name='demo', - shared=True, - metadata=json.dumps({'description': ''}), - created_by=self.user, - organization=self.user) + public_project = Project( + name="demo", + shared=True, + metadata=json.dumps({"description": ""}), + created_by=self.user, + organization=self.user, + ) public_project.save() self.project = public_project data = { - 'owner': 'http://testserver/api/v1/users/%s' - % self.project.organization.username, - 'public': True, - 'public_data': True, - 'description': u'transportation_2011_07_25', - 'downloadable': True, - 'allows_sms': False, - 'encrypted': False, - 'sms_id_string': u'transportation_2011_07_25', - 'id_string': u'transportation_2011_07_25', - 'title': u'transportation_2011_07_25', - 'bamboo_dataset': u'' + "owner": f"http://testserver/api/v1/users/{self.project.organization.username}", + "public": True, + "public_data": True, + "description": "transportation_2011_07_25", + "downloadable": True, + "allows_sms": False, + "encrypted": False, + "sms_id_string": "transportation_2011_07_25", + "id_string": "transportation_2011_07_25", + "title": "transportation_2011_07_25", + "bamboo_dataset": "", } self._publish_xls_form_to_project(publish_data=data, merge=False) - self.assertEquals(self.xform.shared, True) - self.assertEquals(self.xform.shared_data, True) + self.assertEqual(self.xform.shared, True) + self.assertEqual(self.xform.shared_data, True) def test_project_all_users_can_share_remove_themselves(self): self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) - data = {'username': 'alice', 'remove': True} + data = {"username": "alice", "remove": True} for (role_name, role_class) in iteritems(role.ROLES): - ShareProject(self.project, 'alice', role_name).save() + ShareProject(self.project, "alice", role_name).save() - self.assertTrue(role_class.user_has_role(self.user, - self.project)) - self.assertTrue(role_class.user_has_role(self.user, - self.xform)) - data['role'] = role_name + self.assertTrue(role_class.user_has_role(self.user, self.project)) + self.assertTrue(role_class.user_has_role(self.user, self.xform)) + data["role"] = role_name - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 204) - self.assertFalse(role_class.user_has_role(self.user, - self.project)) - self.assertFalse(role_class.user_has_role(self.user, - self.xform)) + self.assertFalse(role_class.user_has_role(self.user, self.project)) + self.assertFalse(role_class.user_has_role(self.user, self.xform)) def test_owner_cannot_remove_self_if_no_other_owner(self): self._project_create() - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) ManagerRole.add(self.user, self.project) - tom_data = {'username': 'tom', 'email': 'tom@localhost.com'} + tom_data = {"username": "tom", "email": "tom@localhost.com"} bob_profile = self._create_user_profile(tom_data) OwnerRole.add(bob_profile.user, self.project) - data = {'username': 'tom', 'remove': True, 'role': 'owner'} + data = {"username": "tom", "remove": True, "role": "owner"} - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 400) - error = {'remove': [u"Project requires at least one owner"]} - self.assertEquals(response.data, error) + error = {"remove": ["Project requires at least one owner"]} + self.assertEqual(response.data, error) - self.assertTrue(OwnerRole.user_has_role(bob_profile.user, - self.project)) + self.assertTrue(OwnerRole.user_has_role(bob_profile.user, self.project)) - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} profile = self._create_user_profile(alice_data) OwnerRole.add(profile.user, self.project) - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) - data = {'username': 'tom', 'remove': True, 'role': 'owner'} + data = {"username": "tom", "remove": True, "role": "owner"} - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 204) - self.assertFalse(OwnerRole.user_has_role(bob_profile.user, - self.project)) + self.assertFalse(OwnerRole.user_has_role(bob_profile.user, self.project)) def test_last_date_modified_changes_when_adding_new_form(self): self._project_create() @@ -1753,22 +1779,20 @@ def test_last_date_modified_changes_when_adding_new_form(self): self.project.refresh_from_db() current_last_date = self.project.date_modified - self.assertNotEquals(last_date, current_last_date) + self.assertNotEqual(last_date, current_last_date) self._make_submissions() self.project.refresh_from_db() - self.assertNotEquals(current_last_date, self.project.date_modified) + self.assertNotEqual(current_last_date, self.project.date_modified) def test_anon_project_form_endpoint(self): self._project_create() self._publish_xls_form_to_project() - view = ProjectViewSet.as_view({ - 'get': 'forms' - }) + view = ProjectViewSet.as_view({"get": "forms"}) - request = self.factory.get('/') + request = self.factory.get("/") response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 404) @@ -1777,16 +1801,13 @@ def test_anon_project_list_endpoint(self): self._project_create() self._publish_xls_form_to_project() - view = ProjectViewSet.as_view({ - 'get': 'list' - }) + view = ProjectViewSet.as_view({"get": "list"}) self.project.shared = True self.project.save() - public_projects = Project.objects.filter( - shared=True).count() + public_projects = Project.objects.filter(shared=True).count() - request = self.factory.get('/') + request = self.factory.get("/") response = view(request) self.assertEqual(response.status_code, 200) @@ -1795,45 +1816,42 @@ def test_anon_project_list_endpoint(self): def test_project_manager_can_delete_xform(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) alice = alice_profile.user projectid = self.project.pk self.assertFalse(ManagerRole.user_has_role(alice, self.project)) - data = {'username': 'alice', 'role': ManagerRole.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) + data = { + "username": "alice", + "role": ManagerRole.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) self.assertTrue(ManagerRole.user_has_role(alice, self.project)) - self.assertTrue(alice.has_perm('delete_xform', self.xform)) + self.assertTrue(alice.has_perm("delete_xform", self.xform)) def test_move_project_owner(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) alice = alice_profile.user projectid = self.project.pk self.assertFalse(OwnerRole.user_has_role(alice, self.project)) - view = ProjectViewSet.as_view({ - 'patch': 'partial_update' - }) + view = ProjectViewSet.as_view({"patch": "partial_update"}) - data_patch = { - 'owner': 'http://testserver/api/v1/users/%s' % alice.username - } - request = self.factory.patch('/', data=data_patch, **self.extra) + data_patch = {"owner": f"http://testserver/api/v1/users/{alice.username}"} + request = self.factory.patch("/", data=data_patch, **self.extra) response = view(request, pk=projectid) # bob cannot move project if he does not have can_add_project project @@ -1842,63 +1860,60 @@ def test_move_project_owner(self): # Give bob permission. ManagerRole.add(self.user, alice_profile) - request = self.factory.patch('/', data=data_patch, **self.extra) + request = self.factory.patch("/", data=data_patch, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 200) self.project.refresh_from_db() - self.assertEquals(self.project.organization, alice) + self.assertEqual(self.project.organization, alice) self.assertTrue(OwnerRole.user_has_role(alice, self.project)) def test_cannot_share_project_to_owner(self): # create project and publish form to project self._publish_xls_form_to_project() - data = {'username': self.user.username, 'role': ManagerRole.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) + data = { + "username": self.user.username, + "role": ManagerRole.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data['username'], [u"Cannot share project" - u" with the owner (bob)"]) + self.assertEqual( + response.data["username"], ["Cannot share project with the owner (bob)"] + ) self.assertTrue(OwnerRole.user_has_role(self.user, self.project)) def test_project_share_readonly(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) projectid = self.project.pk - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': ReadOnlyRole.name} - request = self.factory.put('/', data=data, **self.extra) + data = {"username": "alice", "role": ReadOnlyRole.name} + request = self.factory.put("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.xform)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) perms = role.get_object_users_with_permissions(self.project) for p in perms: - user = p.get('user') + user = p.get("user") if user == alice_profile.user: - r = p.get('role') - self.assertEquals(r, ReadOnlyRole.name) + r = p.get("role") + self.assertEqual(r, ReadOnlyRole.name) def test_move_project_owner_org(self): # create project and publish form to project @@ -1907,19 +1922,16 @@ def test_move_project_owner_org(self): projectid = self.project.pk - view = ProjectViewSet.as_view({ - 'patch': 'partial_update' - }) + view = ProjectViewSet.as_view({"patch": "partial_update"}) old_org = self.project.organization data_patch = { - 'owner': 'http://testserver/api/v1/users/%s' % - self.organization.user.username + "owner": f"http://testserver/api/v1/users/{self.organization.user.username}" } - request = self.factory.patch('/', data=data_patch, **self.extra) + request = self.factory.patch("/", data=data_patch, **self.extra) response = view(request, pk=projectid) - for a in response.data.get('teams'): - self.assertIsNotNone(a.get('role')) + for a in response.data.get("teams"): + self.assertIsNotNone(a.get("role")) self.assertEqual(response.status_code, 200) project = Project.objects.get(pk=projectid) @@ -1929,7 +1941,7 @@ def test_move_project_owner_org(self): def test_project_share_inactive_user(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) # set the user inactive @@ -1939,294 +1951,267 @@ def test_project_share_inactive_user(self): projectid = self.project.pk - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': ReadOnlyRole.name} - request = self.factory.put('/', data=data, **self.extra) + data = {"username": "alice", "role": ReadOnlyRole.name} + request = self.factory.put("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 400) - self.assertIsNone( - cache.get(safe_key(f'{PROJ_OWNER_CACHE}{self.project.pk}'))) + self.assertIsNone(cache.get(safe_key(f"{PROJ_OWNER_CACHE}{self.project.pk}"))) self.assertEqual( response.data, - {'username': [u'The following user(s) is/are not active: alice']}) + {"username": ["The following user(s) is/are not active: alice"]}, + ) - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.xform)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) def test_project_share_remove_inactive_user(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) projectid = self.project.pk - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': ReadOnlyRole.name} - request = self.factory.put('/', data=data, **self.extra) + data = {"username": "alice", "role": ReadOnlyRole.name} + request = self.factory.put("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - self.assertIsNone( - cache.get(safe_key(f'{PROJ_OWNER_CACHE}{self.project.pk}'))) + self.assertIsNone(cache.get(safe_key(f"{PROJ_OWNER_CACHE}{self.project.pk}"))) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.xform)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) # set the user inactive self.assertTrue(alice_profile.user.is_active) alice_profile.user.is_active = False alice_profile.user.save() - data = {'username': 'alice', 'role': ReadOnlyRole.name, "remove": True} - request = self.factory.put('/', data=data, **self.extra) + data = {"username": "alice", "role": ReadOnlyRole.name, "remove": True} + request = self.factory.put("/", data=data, **self.extra) self.assertEqual(response.status_code, 204) - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.xform)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) def test_project_share_readonly_no_downloads(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - tom_data = {'username': 'tom', 'email': 'tom@localhost.com'} + tom_data = {"username": "tom", "email": "tom@localhost.com"} tom_data = self._create_user_profile(tom_data) projectid = self.project.pk self.assertFalse( - ReadOnlyRoleNoDownload.user_has_role(alice_profile.user, - self.project)) + ReadOnlyRoleNoDownload.user_has_role(alice_profile.user, self.project) + ) - data = {'username': 'alice', 'role': ReadOnlyRoleNoDownload.name} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "alice", "role": ReadOnlyRoleNoDownload.name} + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share', - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"post": "share", "get": "retrieve"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - data = {'username': 'tom', 'role': ReadOnlyRole.name} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "tom", "role": ReadOnlyRole.name} + request = self.factory.post("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) # get the users - users = response.data.get('users') + users = response.data.get("users") self.assertEqual(len(users), 3) for user in users: - if user.get('user') == 'bob': - self.assertEquals(user.get('role'), 'owner') - elif user.get('user') == 'alice': - self.assertEquals(user.get('role'), 'readonly-no-download') - elif user.get('user') == 'tom': - self.assertEquals(user.get('role'), 'readonly') + if user.get("user") == "bob": + self.assertEqual(user.get("role"), "owner") + elif user.get("user") == "alice": + self.assertEqual(user.get("role"), "readonly-no-download") + elif user.get("user") == "tom": + self.assertEqual(user.get("role"), "readonly") def test_team_users_in_a_project(self): self._team_create() - project = Project.objects.create(name="Test Project", - organization=self.team.organization, - created_by=self.user, - metadata='{}') + project = Project.objects.create( + name="Test Project", + organization=self.team.organization, + created_by=self.user, + metadata="{}", + ) - chuck_data = {'username': 'chuck', 'email': 'chuck@localhost.com'} + chuck_data = {"username": "chuck", "email": "chuck@localhost.com"} chuck_profile = self._create_user_profile(chuck_data) user_chuck = chuck_profile.user - view = TeamViewSet.as_view({ - 'post': 'share'}) + view = TeamViewSet.as_view({"post": "share"}) - self.assertFalse(EditorRole.user_has_role(user_chuck, - project)) - data = {'role': EditorRole.name, - 'project': project.pk} + self.assertFalse(EditorRole.user_has_role(user_chuck, project)) + data = {"role": EditorRole.name, "project": project.pk} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, pk=self.team.pk) self.assertEqual(response.status_code, 204) tools.add_user_to_team(self.team, user_chuck) self.assertTrue(EditorRole.user_has_role(user_chuck, project)) - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"get": "retrieve"}) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=project.pk) - self.assertIsNotNone(response.data['teams']) - self.assertEquals(3, len(response.data['teams'])) - self.assertEquals(response.data['teams'][2]['role'], 'editor') - self.assertEquals(response.data['teams'][2]['users'][0], - str(chuck_profile.user.username)) + self.assertIsNotNone(response.data["teams"]) + self.assertEqual(3, len(response.data["teams"])) + self.assertEqual(response.data["teams"][2]["role"], "editor") + self.assertEqual( + response.data["teams"][2]["users"][0], str(chuck_profile.user.username) + ) def test_project_accesible_by_admin_created_by_diff_admin(self): self._org_create() # user 1 - chuck_data = {'username': 'chuck', 'email': 'chuck@localhost.com'} + chuck_data = {"username": "chuck", "email": "chuck@localhost.com"} chuck_profile = self._create_user_profile(chuck_data) # user 2 - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - view = OrganizationProfileViewSet.as_view({ - 'post': 'members', - }) + view = OrganizationProfileViewSet.as_view( + { + "post": "members", + } + ) # save the org creator bob = self.user data = json.dumps( - {"username": alice_profile.user.username, - "role": OwnerRole.name}) + {"username": alice_profile.user.username, "role": OwnerRole.name} + ) # create admin 1 request = self.factory.post( - '/', data=data, content_type='application/json', **self.extra) - response = view(request, user='denoinc') + "/", data=data, content_type="application/json", **self.extra + ) + response = view(request, user="denoinc") - self.assertEquals(201, response.status_code) + self.assertEqual(201, response.status_code) data = json.dumps( - {"username": chuck_profile.user.username, - "role": OwnerRole.name}) + {"username": chuck_profile.user.username, "role": OwnerRole.name} + ) # create admin 2 request = self.factory.post( - '/', data=data, content_type='application/json', **self.extra) - response = view(request, user='denoinc') + "/", data=data, content_type="application/json", **self.extra + ) + response = view(request, user="denoinc") - self.assertEquals(201, response.status_code) + self.assertEqual(201, response.status_code) # admin 2 creates a project self.user = chuck_profile.user - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": f"Token {self.user.auth_token}"} data = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' % - self.organization.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False + "name": "demo", + "owner": f"http://testserver/api/v1/users/{self.organization.user.username}", + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, } self._project_create(project_data=data) - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"get": "retrieve"}) # admin 1 tries to access project created by admin 2 self.user = alice_profile.user - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} - request = self.factory.get('/', **self.extra) + self.extra = {"HTTP_AUTHORIZATION": f"Token {self.user.auth_token}"} + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertEquals(200, response.status_code) + self.assertEqual(200, response.status_code) # assert admin can add colaborators - tompoo_data = {'username': 'tompoo', 'email': 'tompoo@localhost.com'} + tompoo_data = {"username": "tompoo", "email": "tompoo@localhost.com"} self._create_user_profile(tompoo_data) - data = {'username': 'tompoo', 'role': ReadOnlyRole.name} - request = self.factory.put('/', data=data, **self.extra) + data = {"username": "tompoo", "role": ReadOnlyRole.name} + request = self.factory.put("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 204) self.user = bob - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % bob.auth_token} + self.extra = {"HTTP_AUTHORIZATION": f"Token {bob.auth_token}"} # remove from admin org data = json.dumps({"username": alice_profile.user.username}) - view = OrganizationProfileViewSet.as_view({ - 'delete': 'members' - }) + view = OrganizationProfileViewSet.as_view({"delete": "members"}) request = self.factory.delete( - '/', data=data, content_type='application/json', **self.extra) - response = view(request, user='denoinc') - self.assertEquals(200, response.status_code) + "/", data=data, content_type="application/json", **self.extra + ) + response = view(request, user="denoinc") + self.assertEqual(200, response.status_code) - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"get": "retrieve"}) self.user = alice_profile.user - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} - request = self.factory.get('/', **self.extra) + self.extra = {"HTTP_AUTHORIZATION": f"Token {self.user.auth_token}"} + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) # user cant access the project removed from org - self.assertEquals(404, response.status_code) + self.assertEqual(404, response.status_code) def test_public_project_on_creation(self): - view = ProjectViewSet.as_view({ - 'post': 'create' - }) + view = ProjectViewSet.as_view({"post": "create"}) data = { - 'name': u'demopublic', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': True + "name": "demopublic", + "owner": f"http://testserver/api/v1/users/{self.user.username}", + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": True, } request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 201) - project = Project.prefetched.filter( - name=data['name'], created_by=self.user)[0] + project = Project.prefetched.filter(name=data["name"], created_by=self.user)[0] self.assertTrue(project.shared) @@ -2235,109 +2220,111 @@ def test_permission_passed_to_dataview_parent_form(self): self._project_create() project1 = self.project self._publish_xls_form_to_project() - data = {'name': u'demo2', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False} + data = { + "name": "demo2", + "owner": f"http://testserver/api/v1/users/{self.user.username}", + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, + } self._project_create(data) project2 = self.project columns = json.dumps(self.xform.get_field_name_xpaths_only()) - data = {'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % project2.pk, - 'columns': columns, - 'query': '[ ]'} + data = { + "name": "My DataView", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{project2.pk}", + "columns": columns, + "query": "[ ]", + } self._create_dataview(data) - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - view = ProjectViewSet.as_view({'put': 'share'}) + view = ProjectViewSet.as_view({"put": "share"}) - data = {'username': 'alice', 'remove': True} + data = {"username": "alice", "remove": True} for (role_name, role_class) in iteritems(role.ROLES): - ShareProject(self.project, 'alice', role_name).save() + ShareProject(self.project, "alice", role_name).save() self.assertFalse(role_class.user_has_role(self.user, project1)) self.assertTrue(role_class.user_has_role(self.user, project2)) self.assertTrue(role_class.user_has_role(self.user, self.xform)) - data['role'] = role_name + data["role"] = role_name - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 204) - self.assertFalse(role_class.user_has_role(self.user, - project1)) - self.assertFalse(role_class.user_has_role(self.user, - self.project)) - self.assertFalse(role_class.user_has_role(self.user, - self.xform)) + self.assertFalse(role_class.user_has_role(self.user, project1)) + self.assertFalse(role_class.user_has_role(self.user, self.project)) + self.assertFalse(role_class.user_has_role(self.user, self.xform)) def test_permission_not_passed_to_dataview_parent_form(self): self._project_create() project1 = self.project self._publish_xls_form_to_project() - data = {'name': u'demo2', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False} + data = { + "name": "demo2", + "owner": f"http://testserver/api/v1/users/{self.user.username}", + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, + } self._project_create(data) project2 = self.project - data = {'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % project2.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[{"column":"age","filter":">","value":"20"},' - '{"column":"age","filter":"<","value":"50"}]'} + data = { + "name": "My DataView", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{project2.pk}", + "columns": '["name", "age", "gender"]', + "query": '[{"column":"age","filter":">","value":"20"},' + '{"column":"age","filter":"<","value":"50"}]', + } self._create_dataview(data) - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - view = ProjectViewSet.as_view({'put': 'share'}) + view = ProjectViewSet.as_view({"put": "share"}) - data = {'username': 'alice', 'remove': True} + data = {"username": "alice", "remove": True} for (role_name, role_class) in iteritems(role.ROLES): - ShareProject(self.project, 'alice', role_name).save() + ShareProject(self.project, "alice", role_name).save() self.assertFalse(role_class.user_has_role(self.user, project1)) self.assertTrue(role_class.user_has_role(self.user, project2)) self.assertFalse(role_class.user_has_role(self.user, self.xform)) - data['role'] = role_name + data["role"] = role_name - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 204) - self.assertFalse(role_class.user_has_role(self.user, - project1)) - self.assertFalse(role_class.user_has_role(self.user, - self.project)) - self.assertFalse(role_class.user_has_role(self.user, - self.xform)) + self.assertFalse(role_class.user_has_role(self.user, project1)) + self.assertFalse(role_class.user_has_role(self.user, self.project)) + self.assertFalse(role_class.user_has_role(self.user, self.xform)) def test_project_share_xform_meta_perms(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) projectid = self.project.pk @@ -2346,59 +2333,56 @@ def test_project_share_xform_meta_perms(self): MetaData.xform_meta_permission(self.xform, data_value=data_value) for role_class in ROLES_ORDERED: - self.assertFalse(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(role_class.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': role_class.name} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "alice", "role": role_class.name} + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - self.assertTrue(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertTrue(role_class.user_has_role(alice_profile.user, self.project)) if role_class in [EditorRole, EditorMinorRole]: self.assertFalse( - EditorRole.user_has_role(alice_profile.user, self.xform)) + EditorRole.user_has_role(alice_profile.user, self.xform) + ) self.assertTrue( - EditorMinorRole.user_has_role(alice_profile.user, - self.xform)) + EditorMinorRole.user_has_role(alice_profile.user, self.xform) + ) - elif role_class in [DataEntryRole, DataEntryMinorRole, - DataEntryOnlyRole]: + elif role_class in [DataEntryRole, DataEntryMinorRole, DataEntryOnlyRole]: self.assertTrue( - DataEntryRole.user_has_role(alice_profile.user, - self.xform)) + DataEntryRole.user_has_role(alice_profile.user, self.xform) + ) else: self.assertTrue( - role_class.user_has_role(alice_profile.user, self.xform)) + role_class.user_has_role(alice_profile.user, self.xform) + ) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_project_share_atomicity(self, mock_send_mail): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) alice = alice_profile.user projectid = self.project.pk role_class = DataEntryOnlyRole - self.assertFalse(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(role_class.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': role_class.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) + data = { + "username": "alice", + "role": role_class.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) @@ -2407,11 +2391,14 @@ def test_project_share_atomicity(self, mock_send_mail): self.assertTrue(role_class.user_has_role(alice, self.project)) self.assertTrue(role_class.user_has_role(alice, self.xform)) - data['remove'] = True - request = self.factory.post('/', data=data, **self.extra) + data["remove"] = True + request = self.factory.post("/", data=data, **self.extra) mock_rm_xform_perms = MagicMock() - with patch('onadata.libs.models.share_project.remove_xform_permissions', mock_rm_xform_perms): # noqa + with patch( + "onadata.libs.models.share_project.remove_xform_permissions", + mock_rm_xform_perms, + ): # noqa mock_rm_xform_perms.side_effect = Exception() with self.assertRaises(Exception): response = view(request, pk=projectid) @@ -2420,7 +2407,7 @@ def test_project_share_atomicity(self, mock_send_mail): self.assertTrue(role_class.user_has_role(alice, self.project)) self.assertTrue(mock_rm_xform_perms.called) - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) # permissions have changed for both project and xform @@ -2429,71 +2416,67 @@ def test_project_share_atomicity(self, mock_send_mail): def test_project_list_by_owner(self): # create project and publish form to project - sluggie_data = {'username': 'sluggie', - 'email': 'sluggie@localhost.com'} + sluggie_data = {"username": "sluggie", "email": "sluggie@localhost.com"} self._login_user_and_profile(sluggie_data) self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) projectid = self.project.pk - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': ReadOnlyRole.name} - request = self.factory.put('/', data=data, **self.extra) + data = {"username": "alice", "role": ReadOnlyRole.name} + request = self.factory.put("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'put': 'share', - 'get': 'list' - }) + view = ProjectViewSet.as_view({"put": "share", "get": "list"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - self.assertIsNone( - cache.get(safe_key(f'{PROJ_OWNER_CACHE}{self.project.pk}'))) + self.assertIsNone(cache.get(safe_key(f"{PROJ_OWNER_CACHE}{self.project.pk}"))) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.xform)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) # Should list collaborators data = {"owner": "sluggie"} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request) - users = response.data[0]['users'] + users = response.data[0]["users"] self.assertEqual(response.status_code, 200) - self.assertIn({'first_name': u'Bob', 'last_name': u'erama', - 'is_org': False, 'role': 'readonly', 'user': u'alice', - 'metadata': {}}, users) + self.assertIn( + { + "first_name": "Bob", + "last_name": "erama", + "is_org": False, + "role": "readonly", + "user": "alice", + "metadata": {}, + }, + users, + ) def test_projects_soft_delete(self): self._project_create() - view = ProjectViewSet.as_view({ - 'get': 'list', - 'delete': 'destroy' - }) + view = ProjectViewSet.as_view({"get": "list", "delete": "destroy"}) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.user = self.user response = view(request) project_id = self.project.pk - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - serializer = BaseProjectSerializer(self.project, - context={'request': request}) + serializer = BaseProjectSerializer(self.project, context={"request": request}) self.assertEqual(response.data, [serializer.data]) - self.assertIn('created_by', list(response.data[0])) + self.assertIn("created_by", list(response.data[0])) - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) request.user = self.user response = view(request, pk=project_id) self.assertEqual(response.status_code, 204) @@ -2501,14 +2484,14 @@ def test_projects_soft_delete(self): self.project = Project.objects.get(pk=project_id) self.assertIsNotNone(self.project.deleted_at) - self.assertTrue('deleted-at' in self.project.name) + self.assertTrue("deleted-at" in self.project.name) self.assertEqual(self.project.deleted_by, self.user) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.user = self.user response = view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertFalse(serializer.data in response.data) @@ -2518,45 +2501,40 @@ def test_project_share_multiple_users(self): Test that the project can be shared to multiple users """ self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - tom_data = {'username': 'tom', 'email': 'tom@localhost.com'} + tom_data = {"username": "tom", "email": "tom@localhost.com"} tom_profile = self._create_user_profile(tom_data) projectid = self.project.pk - self.assertFalse( - ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - self.assertFalse( - ReadOnlyRole.user_has_role(tom_profile.user, self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(tom_profile.user, self.project)) - data = {'username': 'alice,tom', 'role': ReadOnlyRole.name} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "alice,tom", "role": ReadOnlyRole.name} + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share', - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"post": "share", "get": "retrieve"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) # get the users - users = response.data.get('users') + users = response.data.get("users") self.assertEqual(len(users), 3) for user in users: - if user.get('user') == 'bob': - self.assertEquals(user.get('role'), 'owner') + if user.get("user") == "bob": + self.assertEqual(user.get("role"), "owner") else: - self.assertEquals(user.get('role'), 'readonly') + self.assertEqual(user.get("role"), "readonly") - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_sends_mail_on_multi_share(self, mock_send_mail): """ Test that on sharing a projects to multiple users mail is sent to all @@ -2564,109 +2542,96 @@ def test_sends_mail_on_multi_share(self, mock_send_mail): """ # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - tom_data = {'username': 'tom', 'email': 'tom@localhost.com'} + tom_data = {"username": "tom", "email": "tom@localhost.com"} tom_profile = self._create_user_profile(tom_data) projectid = self.project.pk - self.assertFalse( - ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - self.assertFalse( - ReadOnlyRole.user_has_role(tom_profile.user, self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(tom_profile.user, self.project)) - data = {'username': 'alice,tom', 'role': ReadOnlyRole.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) + data = { + "username": "alice,tom", + "role": ReadOnlyRole.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) self.assertTrue(mock_send_mail.called) self.assertEqual(mock_send_mail.call_count, 2) - self.assertTrue( - ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - self.assertTrue( - ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) - self.assertTrue( - ReadOnlyRole.user_has_role(tom_profile.user, self.project)) - self.assertTrue( - ReadOnlyRole.user_has_role(tom_profile.user, self.xform)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) + self.assertTrue(ReadOnlyRole.user_has_role(tom_profile.user, self.project)) + self.assertTrue(ReadOnlyRole.user_has_role(tom_profile.user, self.xform)) def test_project_caching(self): """ Test project viewset caching always keeps the latest version of the project in cache """ - view = ProjectViewSet.as_view({ - 'post': 'forms', - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"post": "forms", "get": "retrieve"}) self._publish_xls_form_to_project() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data['forms']), 1) - self.assertEqual( - response.data['forms'][0]['name'], self.xform.title) + self.assertEqual(len(response.data["forms"]), 1) + self.assertEqual(response.data["forms"][0]["name"], self.xform.title) self.assertEqual( - response.data['forms'][0]['last_submission_time'], - self.xform.time_of_last_submission()) + response.data["forms"][0]["last_submission_time"], + self.xform.time_of_last_submission(), + ) self.assertEqual( - response.data['forms'][0]['num_of_submissions'], - self.xform.num_of_submissions + response.data["forms"][0]["num_of_submissions"], + self.xform.num_of_submissions, ) - self.assertEqual(response.data['num_datasets'], 1) + self.assertEqual(response.data["num_datasets"], 1) # Test on form detail update data returned from project viewset is # updated - form_view = XFormViewSet.as_view({ - 'patch': 'partial_update' - }) - post_data = {'title': 'new_name'} - request = self.factory.patch( - '/', data=post_data, **self.extra) + form_view = XFormViewSet.as_view({"patch": "partial_update"}) + post_data = {"title": "new_name"} + request = self.factory.patch("/", data=post_data, **self.extra) response = form_view(request, pk=self.xform.pk) self.assertEqual(response.status_code, 200) self.xform.refresh_from_db() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data['forms']), 1) - self.assertEqual( - response.data['forms'][0]['name'], self.xform.title) + self.assertEqual(len(response.data["forms"]), 1) + self.assertEqual(response.data["forms"][0]["name"], self.xform.title) self.assertEqual( - response.data['forms'][0]['last_submission_time'], - self.xform.time_of_last_submission()) + response.data["forms"][0]["last_submission_time"], + self.xform.time_of_last_submission(), + ) self.assertEqual( - response.data['forms'][0]['num_of_submissions'], - self.xform.num_of_submissions + response.data["forms"][0]["num_of_submissions"], + self.xform.num_of_submissions, ) - self.assertEqual(response.data['num_datasets'], 1) + self.assertEqual(response.data["num_datasets"], 1) # Test that last_submission_time is updated correctly self._make_submissions() self.xform.refresh_from_db() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data['forms']), 1) - self.assertEqual( - response.data['forms'][0]['name'], self.xform.title) - self.assertIsNotNone(response.data['forms'][0]['last_submission_time']) + self.assertEqual(len(response.data["forms"]), 1) + self.assertEqual(response.data["forms"][0]["name"], self.xform.title) + self.assertIsNotNone(response.data["forms"][0]["last_submission_time"]) returned_date = dateutil.parser.parse( - response.data['forms'][0]['last_submission_time']) - self.assertEqual( - returned_date, - self.xform.time_of_last_submission()) + response.data["forms"][0]["last_submission_time"] + ) + self.assertEqual(returned_date, self.xform.time_of_last_submission()) self.assertEqual( - response.data['forms'][0]['num_of_submissions'], - self.xform.num_of_submissions + response.data["forms"][0]["num_of_submissions"], + self.xform.num_of_submissions, ) - self.assertEqual(response.data['num_datasets'], 1) + self.assertEqual(response.data["num_datasets"], 1) diff --git a/onadata/apps/api/tests/viewsets/test_stats_viewset.py b/onadata/apps/api/tests/viewsets/test_stats_viewset.py index 0ab54e9258..83e395102d 100644 --- a/onadata/apps/api/tests/viewsets/test_stats_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_stats_viewset.py @@ -8,256 +8,258 @@ from onadata.apps.main.tests.test_base import TestBase from onadata.apps.api.viewsets.stats_viewset import StatsViewSet -from onadata.apps.api.viewsets.submissionstats_viewset import\ - SubmissionStatsViewSet +from onadata.apps.api.viewsets.submissionstats_viewset import SubmissionStatsViewSet from onadata.apps.logger.models import XForm from onadata.libs.utils.logger_tools import publish_xml_form, create_instance from onadata.libs.utils.user_auth import get_user_default_project class TestStatsViewSet(TestBase): - def setUp(self): TestBase.setUp(self) self._create_user_and_login() self.factory = RequestFactory() - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} - @patch('onadata.apps.logger.models.instance.submission_time') + @patch("onadata.apps.logger.models.instance.submission_time") def test_submissions_stats(self, mock_time): self._set_mock_time(mock_time) self._publish_transportation_form() self._make_submissions() - view = SubmissionStatsViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = SubmissionStatsViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) formid = self.xform.pk - data = [{ - 'id': formid, - 'id_string': u'transportation_2011_07_25', - 'url': 'http://testserver/api/v1/stats/submissions/%s' % formid - }] + data = [ + { + "id": formid, + "id_string": "transportation_2011_07_25", + "url": "http://testserver/api/v1/stats/submissions/%s" % formid, + } + ] self.assertEqual(response.data, data) - view = SubmissionStatsViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/', **self.extra) + view = SubmissionStatsViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 400) - data = {u'detail': u'Expecting `group` and `name` query parameters.'} + data = {"detail": "Expecting `group` and `name` query parameters."} self.assertEqual(response.data, data) - request = self.factory.get('/?group=_xform_id_string', **self.extra) + request = self.factory.get("/?group=_xform_id_string", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) - data = { - u'count': 4 - } - self.assertDictContainsSubset(data, response.data[0]) + data = {"count": 4} + self.assertDictContainsSubset(data, response.data[0], response.data) - request = self.factory.get('/?group=_submitted_by', **self.extra) + request = self.factory.get("/?group=_submitted_by", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) - data = { - u'count': 4 - } + data = {"count": 4} self.assertDictContainsSubset(data, response.data[0]) - @patch('onadata.apps.logger.models.instance.submission_time') - def test_submissions_stats_with_xform_in_delete_async_queue( - self, mock_time): + @patch("onadata.apps.logger.models.instance.submission_time") + def test_submissions_stats_with_xform_in_delete_async_queue(self, mock_time): self._set_mock_time(mock_time) self._publish_transportation_form() self._make_submissions() - view = SubmissionStatsViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = SubmissionStatsViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) formid = self.xform.pk - data = [{ - 'id': formid, - 'id_string': u'transportation_2011_07_25', - 'url': 'http://testserver/api/v1/stats/submissions/%s' % formid - }] + data = [ + { + "id": formid, + "id_string": "transportation_2011_07_25", + "url": "http://testserver/api/v1/stats/submissions/%s" % formid, + } + ] self.assertEqual(response.data, data) initial_count = len(response.data) self.xform.deleted_at = timezone.now() self.xform.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), initial_count - 1) def test_form_list_select_one_choices_multi_language(self): - paths = [os.path.join( - self.this_directory, 'fixtures', 'good_eats_multilang', x) - for x in ['good_eats_multilang.xlsx', '1.xml']] + paths = [ + os.path.join(self.this_directory, "fixtures", "good_eats_multilang", x) + for x in ["good_eats_multilang.xlsx", "1.xml"] + ] self._publish_xls_file_and_set_xform(paths[0]) self._make_submission(paths[1]) - view = SubmissionStatsViewSet.as_view({'get': 'retrieve'}) + view = SubmissionStatsViewSet.as_view({"get": "retrieve"}) formid = self.xform.pk - request = self.factory.get('/?group=rating', - **self.extra) + request = self.factory.get("/?group=rating", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) - data = [{'count': 1, 'rating': u'Nothing Special'}] + data = [{"count": 1, "rating": "Nothing Special"}] self.assertEqual(data, response.data) def test_form_list_select_one_choices(self): self._tutorial_form_submission() - view = SubmissionStatsViewSet.as_view({'get': 'retrieve'}) + view = SubmissionStatsViewSet.as_view({"get": "retrieve"}) formid = self.xform.pk - request = self.factory.get('/?group=gender', **self.extra) + request = self.factory.get("/?group=gender", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) - data = [ - {'count': 2, 'gender': u'Female'}, - {'count': 1, 'gender': u'Male'} - ] - self.assertEqual(sorted(data, key=lambda k: k['gender']), - sorted(response.data, key=lambda k: k['gender'])) + data = [{"count": 2, "gender": "Female"}, {"count": 1, "gender": "Male"}] + self.assertEqual( + sorted(data, key=lambda k: k["gender"]), + sorted(response.data, key=lambda k: k["gender"]), + ) def test_anon_form_list(self): self._publish_transportation_form() self._make_submissions() - view = SubmissionStatsViewSet.as_view({'get': 'list'}) - request = self.factory.get('/') + view = SubmissionStatsViewSet.as_view({"get": "list"}) + request = self.factory.get("/") response = view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) def _tutorial_form_submission(self): tutorial_folder = os.path.join( - os.path.dirname(__file__), - '..', 'fixtures', 'forms', 'tutorial') - self._publish_xls_file_and_set_xform(os.path.join(tutorial_folder, - 'tutorial.xlsx')) - instance_paths = [os.path.join(tutorial_folder, 'instances', i) - for i in ['1.xml', '2.xml', '3.xml']] + os.path.dirname(__file__), "..", "fixtures", "forms", "tutorial" + ) + self._publish_xls_file_and_set_xform( + os.path.join(tutorial_folder, "tutorial.xlsx") + ) + instance_paths = [ + os.path.join(tutorial_folder, "instances", i) + for i in ["1.xml", "2.xml", "3.xml"] + ] for path in instance_paths: - create_instance(self.user.username, open(path, 'rb'), []) + create_instance(self.user.username, open(path, "rb"), []) self.assertEqual(self.xform.instances.count(), 3) def _contributions_form_submissions(self): count = XForm.objects.count() - path = os.path.join(os.path.dirname(__file__), - '..', 'fixtures', 'forms', 'contributions') - form_path = os.path.join(path, 'contributions.xml') - with open(form_path, encoding='utf-8') as f: + path = os.path.join( + os.path.dirname(__file__), "..", "fixtures", "forms", "contributions" + ) + form_path = os.path.join(path, "contributions.xml") + with open(form_path, encoding="utf-8") as f: xml_file = ContentFile(f.read()) - xml_file.name = 'contributions.xml' + xml_file.name = "contributions.xml" project = get_user_default_project(self.user) self.xform = publish_xml_form(xml_file, self.user, project) self.assertTrue(XForm.objects.count() > count) - instances_path = os.path.join(path, 'instances') + instances_path = os.path.join(path, "instances") for uuid in os.listdir(instances_path): - s_path = os.path.join(instances_path, uuid, 'submission.xml') - create_instance(self.user.username, open(s_path, 'rb'), []) + s_path = os.path.join(instances_path, uuid, "submission.xml") + create_instance(self.user.username, open(s_path, "rb"), []) self.assertEqual(self.xform.instances.count(), 6) def test_median_api(self): self._contributions_form_submissions() - view = StatsViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/?method=median', **self.extra) + view = StatsViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/?method=median", **self.extra) formid = self.xform.pk response = view(request, pk=formid) - data = {u'age': 28.5, u'amount': 1100.0} - self.assertNotEqual(response.get('Cache-Control'), None) + data = {"age": 28.5, "amount": 1100.0} + self.assertNotEqual(response.get("Cache-Control"), None) self.assertDictContainsSubset(data, response.data) def test_mean_api(self): self._contributions_form_submissions() - view = StatsViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/?method=mean', **self.extra) + view = StatsViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/?method=mean", **self.extra) formid = self.xform.pk response = view(request, pk=formid) - data = {u'age': 28.17, u'amount': 1455.0} - self.assertNotEqual(response.get('Cache-Control'), None) + data = {"age": 28.17, "amount": 1455.0} + self.assertNotEqual(response.get("Cache-Control"), None) self.assertDictContainsSubset(data, response.data) def test_mode_api(self): self._contributions_form_submissions() - view = StatsViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/?method=mode', **self.extra) + view = StatsViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/?method=mode", **self.extra) formid = self.xform.pk response = view(request, pk=formid) - data = {u'age': 24, u'amount': 430.0} - self.assertNotEqual(response.get('Cache-Control'), None) + data = {"age": 24, "amount": 430.0} + self.assertNotEqual(response.get("Cache-Control"), None) self.assertDictContainsSubset(data, response.data) def test_range_api(self): self._contributions_form_submissions() - view = StatsViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/?method=range', **self.extra) + view = StatsViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/?method=range", **self.extra) formid = self.xform.pk response = view(request, pk=formid) - data = {u'age': {u'range': 10, u'max': 34, u'min': 24}, - u'amount': {u'range': 2770, u'max': 3200, u'min': 430}} - self.assertNotEqual(response.get('Cache-Control'), None) + data = { + "age": {"range": 10, "max": 34, "min": 24}, + "amount": {"range": 2770, "max": 3200, "min": 430}, + } + self.assertNotEqual(response.get("Cache-Control"), None) self.assertDictContainsSubset(data, response.data) def test_bad_field(self): self._contributions_form_submissions() - view = StatsViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/?method=median&field=INVALID', - **self.extra) + view = StatsViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/?method=median&field=INVALID", **self.extra) formid = self.xform.pk response = view(request, pk=formid) self.assertEqual(response.status_code, 400) def test_all_stats_api(self): self._contributions_form_submissions() - view = StatsViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = StatsViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) formid = self.xform.pk - data = [{ - u'id': formid, - u'id_string': u'contributions', - u'url': u'http://testserver/api/v1/stats/%s' % formid - }] + data = [ + { + "id": formid, + "id_string": "contributions", + "url": "http://testserver/api/v1/stats/%s" % formid, + } + ] self.assertEqual(data, response.data) - view = StatsViewSet.as_view({'get': 'retrieve'}) + view = StatsViewSet.as_view({"get": "retrieve"}) response = view(request, pk=formid) data = {} - data['age'] = { - 'mean': 28.17, - 'median': 28.5, - 'mode': 24, - 'max': 34, - 'min': 24, - 'range': 10 + data["age"] = { + "mean": 28.17, + "median": 28.5, + "mode": 24, + "max": 34, + "min": 24, + "range": 10, } - request = self.factory.get('/?field=age', **self.extra) + request = self.factory.get("/?field=age", **self.extra) age_response = view(request, pk=formid) self.assertEqual(data, age_response.data) - data['amount'] = { - 'mean': 1455, - 'median': 1100.0, - 'mode': 430, - 'max': 3200, - 'min': 430, - 'range': 2770 + data["amount"] = { + "mean": 1455, + "median": 1100.0, + "mode": 430, + "max": 3200, + "min": 430, + "range": 2770, } self.assertDictContainsSubset(data, response.data) def test_wrong_stat_function_api(self): self._contributions_form_submissions() - view = StatsViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/?method=modes', **self.extra) + view = StatsViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/?method=modes", **self.extra) formid = self.xform.pk response = view(request, pk=formid) - self.assertNotEqual(response.get('Cache-Control'), None) - self.assertEquals(response.status_code, 200) + self.assertNotEqual(response.get("Cache-Control"), None) + self.assertEqual(response.status_code, 200) diff --git a/onadata/apps/api/tests/viewsets/test_team_viewset.py b/onadata/apps/api/tests/viewsets/test_team_viewset.py index 2505c42be9..c0e91c9f32 100644 --- a/onadata/apps/api/tests/viewsets/test_team_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_team_viewset.py @@ -282,7 +282,7 @@ def test_team_share_members(self): request = self.factory.get('/', data=get_data, **self.extra) response = view(request) # get the members team - self.assertEquals(response.data[1].get('name'), 'members') + self.assertEqual(response.data[1].get('name'), 'members') teamid = response.data[1].get('teamid') chuck_data = {'username': 'chuck', 'email': 'chuck@localhost.com'} diff --git a/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py b/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py index 9b1f8f7f89..19fdf2b9de 100644 --- a/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py @@ -1,10 +1,14 @@ +# -*- coding: utf-8 -*- +""" +Tests the UserProfileViewSet. +""" +# pylint: disable=too-many-lines import datetime import json import os -from builtins import str -from future.moves.urllib.parse import urlparse, parse_qs +from six.moves.urllib.parse import urlparse, parse_qs -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.cache import cache from django.db.models import signals from django.test.utils import override_settings @@ -18,97 +22,106 @@ from registration.models import RegistrationProfile from rest_framework.authtoken.models import Token -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.viewsets.connect_viewset import ConnectViewSet from onadata.apps.api.viewsets.user_profile_viewset import UserProfileViewSet from onadata.apps.logger.models.instance import Instance from onadata.apps.main.models import UserProfile -from onadata.apps.main.models.user_profile import \ - set_kpi_formbuilder_permissions +from onadata.apps.main.models.user_profile import set_kpi_formbuilder_permissions from onadata.libs.authentication import DigestAuthentication -from onadata.libs.serializers.user_profile_serializer import \ - _get_first_last_names +from onadata.libs.serializers.user_profile_serializer import _get_first_last_names + + +User = get_user_model() def _profile_data(): return { - 'username': u'deno', - 'first_name': u'Dennis', - 'last_name': u'erama', - 'email': u'deno@columbia.edu', - 'city': u'Denoville', - 'country': u'US', - 'organization': u'Dono Inc.', - 'website': u'deno.com', - 'twitter': u'denoerama', - 'require_auth': False, - 'password': 'denodeno', - 'is_org': False, - 'name': u'Dennis erama' + "username": "deno", + "first_name": "Dennis", + "last_name": "erama", + "email": "deno@columbia.edu", + "city": "Denoville", + "country": "US", + "organization": "Dono Inc.", + "website": "deno.com", + "twitter": "denoerama", + "require_auth": False, + "password": "denodeno", + "is_org": False, + "name": "Dennis erama", } +# pylint: disable=attribute-defined-outside-init,too-many-public-methods +# pylint: disable=invalid-name,missing-class-docstring,missing-function-docstring, +# pylint: disable=consider-using-f-string class TestUserProfileViewSet(TestAbstractViewSet): - def setUp(self): - super(self.__class__, self).setUp() - self.view = UserProfileViewSet.as_view({ - 'get': 'list', - 'post': 'create', - 'patch': 'partial_update', - 'put': 'update' - }) + super().setUp() + self.view = UserProfileViewSet.as_view( + { + "get": "list", + "post": "create", + "patch": "partial_update", + "put": "update", + } + ) def tearDown(self): """ Specific to clear cache between tests """ - super(TestUserProfileViewSet, self).tearDown() + super().tearDown() cache.clear() def test_profiles_list(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) data = self.user_profile_data() - del data['metadata'] + del data["metadata"] self.assertEqual(response.data, [data]) def test_user_profile_list(self): request = self.factory.post( - '/api/v1/profiles', data=json.dumps(_profile_data()), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(_profile_data()), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) data = {"users": "bob,deno"} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = self.view(request) deno_profile_data = _profile_data() - deno_profile_data.pop('password', None) - user_deno = User.objects.get(username='deno') - deno_profile_data.update({ - 'id': user_deno.pk, - 'url': 'http://testserver/api/v1/profiles/%s' % user_deno.username, - 'user': 'http://testserver/api/v1/users/%s' % user_deno.username, - 'gravatar': user_deno.profile.gravatar, - 'joined_on': user_deno.date_joined - }) + deno_profile_data.pop("password", None) + user_deno = User.objects.get(username="deno") + deno_profile_data.update( + { + "id": user_deno.pk, + "url": "http://testserver/api/v1/profiles/%s" % user_deno.username, + "user": "http://testserver/api/v1/users/%s" % user_deno.username, + "gravatar": user_deno.profile.gravatar, + "joined_on": user_deno.date_joined, + } + ) self.assertEqual(response.status_code, 200) user_profile_data = self.user_profile_data() - del user_profile_data['metadata'] + del user_profile_data["metadata"] self.assertEqual( - sorted([dict(d) for d in response.data], key=lambda x: x['id']), - sorted([user_profile_data, deno_profile_data], - key=lambda x: x['id'])) + sorted([dict(d) for d in response.data], key=lambda x: x["id"]), + sorted([user_profile_data, deno_profile_data], key=lambda x: x["id"]), + ) self.assertEqual(len(response.data), 2) # Inactive user not in list @@ -117,36 +130,39 @@ def test_user_profile_list(self): response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) - self.assertNotIn(user_deno.pk, [user['id'] for user in response.data]) + self.assertNotIn(user_deno.pk, [user["id"] for user in response.data]) def test_user_profile_list_with_and_without_users_param(self): request = self.factory.post( - '/api/v1/profiles', data=json.dumps(_profile_data()), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(_profile_data()), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) # anonymous user gets empty response - request = self.factory.get('/') + request = self.factory.get("/") response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) # authenicated user without users query param only gets his/her profile - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) user_profile_data = self.user_profile_data() - del user_profile_data['metadata'] + del user_profile_data["metadata"] self.assertDictEqual(user_profile_data, response.data[0]) # authenicated user with blank users query param only gets his/her # profile data = {"users": ""} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) @@ -155,128 +171,131 @@ def test_user_profile_list_with_and_without_users_param(self): # authenicated user with comma separated usernames as users query param # value gets profiles of the usernames provided data = {"users": "bob,deno"} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = self.view(request) deno_profile_data = _profile_data() - deno_profile_data.pop('password', None) - user_deno = User.objects.get(username='deno') - deno_profile_data.update({ - 'id': user_deno.pk, - 'url': 'http://testserver/api/v1/profiles/%s' % user_deno.username, - 'user': 'http://testserver/api/v1/users/%s' % user_deno.username, - 'gravatar': user_deno.profile.gravatar, - 'joined_on': user_deno.date_joined - }) + deno_profile_data.pop("password", None) + user_deno = User.objects.get(username="deno") + deno_profile_data.update( + { + "id": user_deno.pk, + "url": "http://testserver/api/v1/profiles/%s" % user_deno.username, + "user": "http://testserver/api/v1/users/%s" % user_deno.username, + "gravatar": user_deno.profile.gravatar, + "joined_on": user_deno.date_joined, + } + ) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) self.assertEqual( - [dict(i) for i in response.data], - [user_profile_data, deno_profile_data] + [dict(i) for i in response.data], [user_profile_data, deno_profile_data] ) def test_profiles_get(self): """Test get user profile""" - view = UserProfileViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = UserProfileViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data, {'detail': 'Expected URL keyword argument `user`.'}) + response.data, {"detail": "Expected URL keyword argument `user`."} + ) # by username - response = view(request, user='bob') - self.assertNotEqual(response.get('Cache-Control'), None) + response = view(request, user="bob") + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.user_profile_data()) # by username mixed case - response = view(request, user='BoB') + response = view(request, user="BoB") self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.data, self.user_profile_data()) # by pk response = view(request, user=self.user.pk) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.user_profile_data()) def test_profiles_get_anon(self): - view = UserProfileViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/') + view = UserProfileViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/") response = view(request) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data, {'detail': 'Expected URL keyword argument `user`.'}) - request = self.factory.get('/') - response = view(request, user='bob') + response.data, {"detail": "Expected URL keyword argument `user`."} + ) + request = self.factory.get("/") + response = view(request, user="bob") data = self.user_profile_data() - del data['email'] - del data['metadata'] + del data["email"] + del data["metadata"] self.assertEqual(response.status_code, 200) self.assertEqual(response.data, data) - self.assertNotIn('email', response.data) + self.assertNotIn("email", response.data) def test_profiles_get_org_anon(self): self._org_create() self.client.logout() - view = UserProfileViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/') - response = view(request, user=self.company_data['org']) + view = UserProfileViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/") + response = view(request, user=self.company_data["org"]) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['first_name'], - self.company_data['name']) - self.assertIn('is_org', response.data) - self.assertEqual(response.data['is_org'], True) + self.assertEqual(response.data["first_name"], self.company_data["name"]) + self.assertIn("is_org", response.data) + self.assertEqual(response.data["is_org"], True) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) @override_settings(ENABLE_EMAIL_VERIFICATION=True) @patch( - ('onadata.libs.serializers.user_profile_serializer.' - 'send_verification_email.delay') + ( + "onadata.libs.serializers.user_profile_serializer." + "send_verification_email.delay" + ) ) def test_profile_create(self, mock_send_verification_email): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() - del data['name'] + del data["name"] request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - password = data['password'] - del data['password'] - profile = UserProfile.objects.get(user__username=data['username']) - data['id'] = profile.user.pk - data['gravatar'] = profile.gravatar - data['url'] = 'http://testserver/api/v1/profiles/deno' - data['user'] = 'http://testserver/api/v1/users/deno' - data['metadata'] = {} - data['metadata']['last_password_edit'] = \ - profile.metadata['last_password_edit'] - data['joined_on'] = profile.user.date_joined - data['name'] = "%s %s" % ('Dennis', 'erama') + password = data["password"] + del data["password"] + profile = UserProfile.objects.get(user__username=data["username"]) + data["id"] = profile.user.pk + data["gravatar"] = profile.gravatar + data["url"] = "http://testserver/api/v1/profiles/deno" + data["user"] = "http://testserver/api/v1/users/deno" + data["metadata"] = {} + data["metadata"]["last_password_edit"] = profile.metadata["last_password_edit"] + data["joined_on"] = profile.user.date_joined + data["name"] = "%s %s" % ("Dennis", "erama") self.assertEqual(response.data, data) self.assertTrue(mock_send_verification_email.called) - user = User.objects.get(username='deno') + user = User.objects.get(username="deno") self.assertTrue(user.is_active) self.assertTrue(user.check_password(password), password) def _create_user_using_profiles_endpoint(self, data): request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) @@ -286,103 +305,93 @@ def test_return_204_if_email_verification_variables_are_not_set(self): data = _profile_data() self._create_user_using_profiles_endpoint(data) - view = UserProfileViewSet.as_view({'get': 'verify_email', - 'post': 'send_verification_email'}) - rp = RegistrationProfile.objects.get( - user__username=data.get('username') + view = UserProfileViewSet.as_view( + {"get": "verify_email", "post": "send_verification_email"} ) - _data = {'verification_key': rp.activation_key} - request = self.factory.get('/', data=_data, **self.extra) + rp = RegistrationProfile.objects.get(user__username=data.get("username")) + _data = {"verification_key": rp.activation_key} + request = self.factory.get("/", data=_data, **self.extra) response = view(request) - self.assertEquals(response.status_code, 204) + self.assertEqual(response.status_code, 204) - data = {'username': data.get('username')} - user = User.objects.get(username=data.get('username')) - extra = {'HTTP_AUTHORIZATION': 'Token %s' % user.auth_token} - request = self.factory.post('/', data=data, **extra) + data = {"username": data.get("username")} + user = User.objects.get(username=data.get("username")) + extra = {"HTTP_AUTHORIZATION": "Token %s" % user.auth_token} + request = self.factory.post("/", data=data, **extra) response = view(request) - self.assertEquals(response.status_code, 204) + self.assertEqual(response.status_code, 204) @override_settings(ENABLE_EMAIL_VERIFICATION=True) def test_verification_key_is_valid(self): data = _profile_data() self._create_user_using_profiles_endpoint(data) - view = UserProfileViewSet.as_view({'get': 'verify_email'}) - rp = RegistrationProfile.objects.get( - user__username=data.get('username') - ) - _data = {'verification_key': rp.activation_key} - request = self.factory.get('/', data=_data) + view = UserProfileViewSet.as_view({"get": "verify_email"}) + rp = RegistrationProfile.objects.get(user__username=data.get("username")) + _data = {"verification_key": rp.activation_key} + request = self.factory.get("/", data=_data) response = view(request) - self.assertEquals(response.status_code, 200) - self.assertIn('is_email_verified', response.data) - self.assertIn('username', response.data) - self.assertTrue(response.data.get('is_email_verified')) - self.assertEquals( - response.data.get('username'), data.get('username') - ) + self.assertEqual(response.status_code, 200) + self.assertIn("is_email_verified", response.data) + self.assertIn("username", response.data) + self.assertTrue(response.data.get("is_email_verified")) + self.assertEqual(response.data.get("username"), data.get("username")) - up = UserProfile.objects.get(user__username=data.get('username')) - self.assertIn('is_email_verified', up.metadata) - self.assertTrue(up.metadata.get('is_email_verified')) + up = UserProfile.objects.get(user__username=data.get("username")) + self.assertIn("is_email_verified", up.metadata) + self.assertTrue(up.metadata.get("is_email_verified")) @override_settings(ENABLE_EMAIL_VERIFICATION=True) def test_verification_key_is_valid_with_redirect_url_set(self): data = _profile_data() self._create_user_using_profiles_endpoint(data) - view = UserProfileViewSet.as_view({'get': 'verify_email'}) - rp = RegistrationProfile.objects.get( - user__username=data.get('username') - ) + view = UserProfileViewSet.as_view({"get": "verify_email"}) + rp = RegistrationProfile.objects.get(user__username=data.get("username")) _data = { - 'verification_key': rp.activation_key, - 'redirect_url': 'http://red.ir.ect' + "verification_key": rp.activation_key, + "redirect_url": "http://red.ir.ect", } - request = self.factory.get('/', data=_data) + request = self.factory.get("/", data=_data) response = view(request) - self.assertEquals(response.status_code, 302) - self.assertIn('is_email_verified', response.url) - self.assertIn('username', response.url) + self.assertEqual(response.status_code, 302) + self.assertIn("is_email_verified", response.url) + self.assertIn("username", response.url) string_query_params = urlparse(response.url).query dict_query_params = parse_qs(string_query_params) - self.assertEquals(dict_query_params.get( - 'is_email_verified'), ['True']) - self.assertEquals( - dict_query_params.get('username'), - [data.get('username')] - ) + self.assertEqual(dict_query_params.get("is_email_verified"), ["True"]) + self.assertEqual(dict_query_params.get("username"), [data.get("username")]) - up = UserProfile.objects.get(user__username=data.get('username')) - self.assertIn('is_email_verified', up.metadata) - self.assertTrue(up.metadata.get('is_email_verified')) + up = UserProfile.objects.get(user__username=data.get("username")) + self.assertIn("is_email_verified", up.metadata) + self.assertTrue(up.metadata.get("is_email_verified")) @override_settings(ENABLE_EMAIL_VERIFICATION=True) @patch( - ('onadata.apps.api.viewsets.user_profile_viewset.' - 'send_verification_email.delay') + ( + "onadata.apps.api.viewsets.user_profile_viewset." + "send_verification_email.delay" + ) ) - def test_sending_verification_email_succeeds( - self, mock_send_verification_email): + def test_sending_verification_email_succeeds(self, mock_send_verification_email): data = _profile_data() self._create_user_using_profiles_endpoint(data) - data = {'username': data.get('username')} - view = UserProfileViewSet.as_view({'post': 'send_verification_email'}) + data = {"username": data.get("username")} + view = UserProfileViewSet.as_view({"post": "send_verification_email"}) - user = User.objects.get(username=data.get('username')) - extra = {'HTTP_AUTHORIZATION': 'Token %s' % user.auth_token} - request = self.factory.post('/', data=data, **extra) + user = User.objects.get(username=data.get("username")) + extra = {"HTTP_AUTHORIZATION": "Token %s" % user.auth_token} + request = self.factory.post("/", data=data, **extra) response = view(request) self.assertTrue(mock_send_verification_email.called) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data, "Verification email has been sent") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data, "Verification email has been sent") - user = User.objects.get(username=data.get('username')) - self.assertFalse(user.profile.metadata.get('is_email_verified')) + user = User.objects.get(username=data.get("username")) + self.assertFalse(user.profile.metadata.get("is_email_verified")) @override_settings(VERIFIED_KEY_TEXT=None) @override_settings(ENABLE_EMAIL_VERIFICATION=True) @@ -390,15 +399,13 @@ def test_sending_verification_email_fails(self): data = _profile_data() self._create_user_using_profiles_endpoint(data) - view = UserProfileViewSet.as_view({'post': 'send_verification_email'}) + view = UserProfileViewSet.as_view({"post": "send_verification_email"}) # trigger permission error when username of requesting user is # different from username in post details - request = self.factory.post('/', - data={'username': 'None'}, - **self.extra) + request = self.factory.post("/", data={"username": "None"}, **self.extra) response = view(request) - self.assertEquals(response.status_code, 403) + self.assertEqual(response.status_code, 403) @override_settings(REQUIRE_ODK_AUTHENTICATION=True) def test_profile_require_auth(self): @@ -406,28 +413,34 @@ def test_profile_require_auth(self): Test profile require_auth is True when REQUIRE_ODK_AUTHENTICATION is set to True. """ - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() - del data['name'] + del data["name"] request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - self.assertTrue(response.data.get('require_auth')) + self.assertTrue(response.data.get("require_auth")) def test_profile_create_without_last_name(self): data = { - 'username': u'deno', - 'first_name': u'Dennis', - 'email': u'deno@columbia.edu', + "username": "deno", + "first_name": "Dennis", + "email": "deno@columbia.edu", } request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) @@ -436,127 +449,136 @@ def test_disallow_profile_create_w_same_username(self): self._create_user_using_profiles_endpoint(data) request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 400) - self.assertTrue( - 'deno already exists' in response.data['username'][0]) + self.assertTrue("deno already exists" in response.data["username"][0]) def test_profile_create_with_malfunctioned_email(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = { - 'username': u'nguyenquynh', - 'first_name': u'Nguy\u1ec5n Th\u1ecb', - 'last_name': u'Di\u1ec5m Qu\u1ef3nh', - 'email': u'onademo0+nguyenquynh@gmail.com\ufeff', - 'city': u'Denoville', - 'country': u'US', - 'organization': u'Dono Inc.', - 'website': u'nguyenquynh.com', - 'twitter': u'nguyenquynh', - 'require_auth': False, - 'password': u'onademo', - 'is_org': False, + "username": "nguyenquynh", + "first_name": "Nguy\u1ec5n Th\u1ecb", + "last_name": "Di\u1ec5m Qu\u1ef3nh", + "email": "onademo0+nguyenquynh@gmail.com\ufeff", + "city": "Denoville", + "country": "US", + "organization": "Dono Inc.", + "website": "nguyenquynh.com", + "twitter": "nguyenquynh", + "require_auth": False, + "password": "onademo", + "is_org": False, } request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - password = data['password'] - del data['password'] - - profile = UserProfile.objects.get(user__username=data['username']) - data['id'] = profile.user.pk - data['gravatar'] = profile.gravatar - data['url'] = 'http://testserver/api/v1/profiles/nguyenquynh' - data['user'] = 'http://testserver/api/v1/users/nguyenquynh' - data['metadata'] = {} - data['metadata']['last_password_edit'] = \ - profile.metadata['last_password_edit'] - data['joined_on'] = profile.user.date_joined - data['name'] = "%s %s" % ( - u'Nguy\u1ec5n Th\u1ecb', u'Di\u1ec5m Qu\u1ef3nh') + password = data["password"] + del data["password"] + + profile = UserProfile.objects.get(user__username=data["username"]) + data["id"] = profile.user.pk + data["gravatar"] = profile.gravatar + data["url"] = "http://testserver/api/v1/profiles/nguyenquynh" + data["user"] = "http://testserver/api/v1/users/nguyenquynh" + data["metadata"] = {} + data["metadata"]["last_password_edit"] = profile.metadata["last_password_edit"] + data["joined_on"] = profile.user.date_joined + data["name"] = "%s %s" % ("Nguy\u1ec5n Th\u1ecb", "Di\u1ec5m Qu\u1ef3nh") self.assertEqual(response.data, data) - user = User.objects.get(username='nguyenquynh') + user = User.objects.get(username="nguyenquynh") self.assertTrue(user.is_active) self.assertTrue(user.check_password(password), password) def test_profile_create_with_invalid_username(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() - data['username'] = u'de' - del data['name'] + data["username"] = "de" + del data["name"] request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data.get('username'), - [u'Ensure this field has at least 3 characters.']) + response.data.get("username"), + ["Ensure this field has at least 3 characters."], + ) def test_profile_create_anon(self): data = _profile_data() request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json") + "/api/v1/profiles", data=json.dumps(data), content_type="application/json" + ) response = self.view(request) self.assertEqual(response.status_code, 201) - del data['password'] - del data['email'] - profile = UserProfile.objects.get(user__username=data['username']) - data['id'] = profile.user.pk - data['gravatar'] = profile.gravatar - data['url'] = 'http://testserver/api/v1/profiles/deno' - data['user'] = 'http://testserver/api/v1/users/deno' - data['metadata'] = {} - data['metadata']['last_password_edit'] = \ - profile.metadata['last_password_edit'] - data['joined_on'] = profile.user.date_joined + del data["password"] + del data["email"] + profile = UserProfile.objects.get(user__username=data["username"]) + data["id"] = profile.user.pk + data["gravatar"] = profile.gravatar + data["url"] = "http://testserver/api/v1/profiles/deno" + data["user"] = "http://testserver/api/v1/users/deno" + data["metadata"] = {} + data["metadata"]["last_password_edit"] = profile.metadata["last_password_edit"] + data["joined_on"] = profile.user.date_joined self.assertEqual(response.data, data) - self.assertNotIn('email', response.data) + self.assertNotIn("email", response.data) def test_profile_create_missing_name_field(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() - del data['first_name'] - del data['name'] + del data["first_name"] + del data["name"] request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) response.render() - self.assertContains(response, - 'Either name or first_name should be provided', - status_code=400) + self.assertContains( + response, "Either name or first_name should be provided", status_code=400 + ) def test_split_long_name_to_first_name_and_last_name(self): - name = "(CPLTGL) Centre Pour la Promotion de la Liberte D'Expression "\ + name = ( + "(CPLTGL) Centre Pour la Promotion de la Liberte D'Expression " "et de la Tolerance Dans La Region de" + ) first_name, last_name = _get_first_last_names(name) self.assertEqual(first_name, "(CPLTGL) Centre Pour la Promot") self.assertEqual(last_name, "ion de la Liberte D'Expression") def test_partial_updates(self): - self.assertEqual(self.user.profile.country, u'US') - country = u'KE' - username = 'george' - metadata = {u'computer': u'mac'} + self.assertEqual(self.user.profile.country, "US") + country = "KE" + username = "george" + metadata = {"computer": "mac"} json_metadata = json.dumps(metadata) - data = {'username': username, - 'country': country, - 'metadata': json_metadata} - request = self.factory.patch('/', data=data, **self.extra) + data = {"username": username, "country": country, "metadata": json_metadata} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) @@ -566,15 +588,12 @@ def test_partial_updates(self): def test_partial_updates_empty_metadata(self): profile = UserProfile.objects.get(user=self.user) - profile.metadata = dict() + profile.metadata = {} profile.save() - metadata = {u"zebra": {u"key1": "value1", u"key2": "value2"}} + metadata = {"zebra": {"key1": "value1", "key2": "value2"}} json_metadata = json.dumps(metadata) - data = { - 'metadata': json_metadata, - 'overwrite': 'false' - } - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": json_metadata, "overwrite": "false"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) @@ -582,505 +601,541 @@ def test_partial_updates_empty_metadata(self): def test_partial_updates_too_long(self): # the max field length for username is 30 in django - username = 'a' * 31 - data = {'username': username} - request = self.factory.patch('/', data=data, **self.extra) + username = "a" * 31 + data = {"username": username} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 400) self.assertEqual( response.data, - {'username': - [u'Ensure this field has no more than 30 characters.']}) + {"username": ["Ensure this field has no more than 30 characters."]}, + ) self.assertNotEqual(profile.user.username, username) def test_partial_update_metadata_field(self): - metadata = {u"zebra": {u"key1": "value1", u"key2": "value2"}} + metadata = {"zebra": {"key1": "value1", "key2": "value2"}} json_metadata = json.dumps(metadata) data = { - 'metadata': json_metadata, + "metadata": json_metadata, } - request = self.factory.patch('/', data=data, **self.extra) + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) self.assertEqual(profile.metadata, metadata) # create a new key/value object if it doesn't exist - data = { - 'metadata': '{"zebra": {"key3": "value3"}}', - 'overwrite': u'false' - } - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": '{"zebra": {"key3": "value3"}}', "overwrite": "false"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) self.assertEqual( - profile.metadata, {u"zebra": { - u"key1": "value1", u"key2": "value2", u"key3": "value3"}}) + profile.metadata, + {"zebra": {"key1": "value1", "key2": "value2", "key3": "value3"}}, + ) # update an existing key/value object - data = { - 'metadata': '{"zebra": {"key2": "second"}}', 'overwrite': u'false'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": '{"zebra": {"key2": "second"}}', "overwrite": "false"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) self.assertEqual( - profile.metadata, {u"zebra": { - u"key1": "value1", u"key2": "second", u"key3": "value3"}}) + profile.metadata, + {"zebra": {"key1": "value1", "key2": "second", "key3": "value3"}}, + ) # add a new key/value object if the key doesn't exist - data = { - 'metadata': '{"animal": "donkey"}', 'overwrite': u'false'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": '{"animal": "donkey"}', "overwrite": "false"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) self.assertEqual( - profile.metadata, { - u"zebra": { - u"key1": "value1", u"key2": "second", u"key3": "value3"}, - u'animal': u'donkey'}) + profile.metadata, + { + "zebra": {"key1": "value1", "key2": "second", "key3": "value3"}, + "animal": "donkey", + }, + ) # don't pass overwrite param - data = {'metadata': '{"b": "caah"}'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": '{"b": "caah"}'} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) - self.assertEqual( - profile.metadata, {u'b': u'caah'}) + self.assertEqual(profile.metadata, {"b": "caah"}) # pass 'overwrite' param whose value isn't false - data = {'metadata': '{"b": "caah"}', 'overwrite': u'falsey'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": '{"b": "caah"}', "overwrite": "falsey"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) - self.assertEqual( - profile.metadata, {u'b': u'caah'}) + self.assertEqual(profile.metadata, {"b": "caah"}) def test_put_update(self): data = _profile_data() # create profile request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) # edit username with existing different user's username - data['username'] = 'bob' + data["username"] = "bob" request = self.factory.put( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) - response = self.view(request, user='deno') + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) + response = self.view(request, user="deno") self.assertEqual(response.status_code, 400) # update - data['username'] = 'roger' - data['city'] = 'Nairobi' + data["username"] = "roger" + data["city"] = "Nairobi" request = self.factory.put( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) - response = self.view(request, user='deno') + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) + response = self.view(request, user="deno") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['city'], data['city']) + self.assertEqual(response.data["city"], data["city"]) def test_profile_create_mixed_case(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - del data['password'] - profile = UserProfile.objects.get( - user__username=data['username'].lower()) - data['id'] = profile.user.pk - data['gravatar'] = str(profile.gravatar) - data['url'] = 'http://testserver/api/v1/profiles/deno' - data['user'] = 'http://testserver/api/v1/users/deno' - data['username'] = u'deno' - data['metadata'] = {} - data['metadata']['last_password_edit'] = \ - profile.metadata['last_password_edit'] - data['joined_on'] = profile.user.date_joined + del data["password"] + profile = UserProfile.objects.get(user__username=data["username"].lower()) + data["id"] = profile.user.pk + data["gravatar"] = str(profile.gravatar) + data["url"] = "http://testserver/api/v1/profiles/deno" + data["user"] = "http://testserver/api/v1/users/deno" + data["username"] = "deno" + data["metadata"] = {} + data["metadata"]["last_password_edit"] = profile.metadata["last_password_edit"] + data["joined_on"] = profile.user.date_joined self.assertEqual(response.data, data) - data['username'] = u'deno' - data['joined_on'] = str(profile.user.date_joined) + data["username"] = "deno" + data["joined_on"] = str(profile.user.date_joined) request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 400) - self.assertIn("%s already exists" % - data['username'], response.data['username']) + self.assertIn("%s already exists" % data["username"], response.data["username"]) def test_change_password(self): - view = UserProfileViewSet.as_view( - {'post': 'change_password'}) + view = UserProfileViewSet.as_view({"post": "change_password"}) current_password = "bobbob" new_password = "bobbob1" old_token = Token.objects.get(user=self.user).key - post_data = {'current_password': current_password, - 'new_password': new_password} + post_data = {"current_password": current_password, "new_password": new_password} - request = self.factory.post('/', data=post_data, **self.extra) - response = view(request, user='bob') + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request, user="bob") now = timezone.now().isoformat() user = User.objects.get(username__iexact=self.user.username) user_profile = UserProfile.objects.get(user_id=user.id) self.assertEqual(response.status_code, 200) self.assertEqual( - type(parse_datetime(user_profile.metadata['last_password_edit'])), - type(parse_datetime(now))) + type(parse_datetime(user_profile.metadata["last_password_edit"])), + type(parse_datetime(now)), + ) self.assertTrue(user.check_password(new_password)) - self.assertIn('access_token', response.data) - self.assertIn('temp_token', response.data) - self.assertEqual(response.data['username'], self.user.username) - self.assertNotEqual(response.data['access_token'], old_token) + self.assertIn("access_token", response.data) + self.assertIn("temp_token", response.data) + self.assertEqual(response.data["username"], self.user.username) + self.assertNotEqual(response.data["access_token"], old_token) # Assert requests made with the old tokens are rejected - post_data = { - 'current_password': new_password, - 'new_password': 'random'} - request = self.factory.post('/', data=post_data, **self.extra) - response = view(request, user='bob') + post_data = {"current_password": new_password, "new_password": "random"} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request, user="bob") self.assertEqual(response.status_code, 401) def test_change_password_wrong_current_password(self): - view = UserProfileViewSet.as_view( - {'post': 'change_password'}) + view = UserProfileViewSet.as_view({"post": "change_password"}) current_password = "wrong_pass" new_password = "bobbob1" - post_data = {'current_password': current_password, - 'new_password': new_password} + post_data = {"current_password": current_password, "new_password": new_password} - request = self.factory.post('/', data=post_data, **self.extra) - response = view(request, user='bob') + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request, user="bob") user = User.objects.get(username__iexact=self.user.username) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data, - "Invalid password. You have 9 attempts left.") + response.data, {"error": "Invalid password. You have 9 attempts left."} + ) self.assertFalse(user.check_password(new_password)) def test_profile_create_with_name(self): data = { - 'username': u'deno', - 'name': u'Dennis deno', - 'email': u'deno@columbia.edu', - 'city': u'Denoville', - 'country': u'US', - 'organization': u'Dono Inc.', - 'website': u'deno.com', - 'twitter': u'denoerama', - 'require_auth': False, - 'password': 'denodeno', - 'is_org': False, + "username": "deno", + "name": "Dennis deno", + "email": "deno@columbia.edu", + "city": "Denoville", + "country": "US", + "organization": "Dono Inc.", + "website": "deno.com", + "twitter": "denoerama", + "require_auth": False, + "password": "denodeno", + "is_org": False, } request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - del data['password'] - profile = UserProfile.objects.get(user__username=data['username']) - data['id'] = profile.user.pk - data['first_name'] = 'Dennis' - data['last_name'] = 'deno' - data['gravatar'] = profile.gravatar - data['url'] = 'http://testserver/api/v1/profiles/deno' - data['user'] = 'http://testserver/api/v1/users/deno' - data['metadata'] = {} - data['metadata']['last_password_edit'] = \ - profile.metadata['last_password_edit'] - data['joined_on'] = profile.user.date_joined + del data["password"] + profile = UserProfile.objects.get(user__username=data["username"]) + data["id"] = profile.user.pk + data["first_name"] = "Dennis" + data["last_name"] = "deno" + data["gravatar"] = profile.gravatar + data["url"] = "http://testserver/api/v1/profiles/deno" + data["user"] = "http://testserver/api/v1/users/deno" + data["metadata"] = {} + data["metadata"]["last_password_edit"] = profile.metadata["last_password_edit"] + data["joined_on"] = profile.user.date_joined self.assertEqual(response.data, data) - user = User.objects.get(username='deno') + user = User.objects.get(username="deno") self.assertTrue(user.is_active) def test_twitter_username_validation(self): data = { - 'username': u'deno', - 'name': u'Dennis deno', - 'email': u'deno@columbia.edu', - 'city': u'Denoville', - 'country': u'US', - 'organization': u'Dono Inc.', - 'website': u'deno.com', - 'twitter': u'denoerama', - 'require_auth': False, - 'password': 'denodeno', - 'is_org': False, + "username": "deno", + "name": "Dennis deno", + "email": "deno@columbia.edu", + "city": "Denoville", + "country": "US", + "organization": "Dono Inc.", + "website": "deno.com", + "twitter": "denoerama", + "require_auth": False, + "password": "denodeno", + "is_org": False, } request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - data['twitter'] = 'denoerama' + data["twitter"] = "denoerama" data = { - 'username': u'deno', - 'name': u'Dennis deno', - 'email': u'deno@columbia.edu', - 'city': u'Denoville', - 'country': u'US', - 'organization': u'Dono Inc.', - 'website': u'deno.com', - 'twitter': u'denoeramaddfsdsl8729320392ujijdswkp--22kwklskdsjs', - 'require_auth': False, - 'password': 'denodeno', - 'is_org': False, + "username": "deno", + "name": "Dennis deno", + "email": "deno@columbia.edu", + "city": "Denoville", + "country": "US", + "organization": "Dono Inc.", + "website": "deno.com", + "twitter": "denoeramaddfsdsl8729320392ujijdswkp--22kwklskdsjs", + "require_auth": False, + "password": "denodeno", + "is_org": False, } request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data['twitter'], - [u'Invalid twitter username {}'.format(data['twitter'])] + response.data["twitter"], + ["Invalid twitter username {}".format(data["twitter"])], ) - user = User.objects.get(username='deno') + user = User.objects.get(username="deno") self.assertTrue(user.is_active) def test_put_patch_method_on_names(self): data = _profile_data() # create profile request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) # update - data['first_name'] = 'Tom' - del data['name'] + data["first_name"] = "Tom" + del data["name"] request = self.factory.put( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) - response = self.view(request, user='deno') + response = self.view(request, user="deno") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['first_name'], data['first_name']) + self.assertEqual(response.data["first_name"], data["first_name"]) - first_name = u'Henry' - last_name = u'Thierry' + first_name = "Henry" + last_name = "Thierry" - data = {'first_name': first_name, 'last_name': last_name} - request = self.factory.patch('/', data=data, **self.extra) + data = {"first_name": first_name, "last_name": last_name} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['first_name'], data['first_name']) - self.assertEqual(response.data['last_name'], data['last_name']) + self.assertEqual(response.data["first_name"], data["first_name"]) + self.assertEqual(response.data["last_name"], data["last_name"]) - @patch('django.core.mail.EmailMultiAlternatives.send') + @patch("django.core.mail.EmailMultiAlternatives.send") def test_send_email_activation_api(self, mock_send_mail): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() - del data['name'] + del data["name"] request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) # Activation email not sent self.assertFalse(mock_send_mail.called) - user = User.objects.get(username='deno') + user = User.objects.get(username="deno") self.assertTrue(user.is_active) def test_partial_update_without_password_fails(self): - data = {'email': 'user@example.com'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"email": "user@example.com"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) self.assertEqual(response.status_code, 400) self.assertEqual( - [u'Your password is required when updating your email address.'], - response.data) + ["Your password is required when updating your email address."], + response.data, + ) def test_partial_update_with_invalid_email_fails(self): - data = {'email': 'user@example'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"email": "user@example"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) self.assertEqual(response.status_code, 400) @patch( - ('onadata.libs.serializers.user_profile_serializer.' - 'send_verification_email.delay') + ( + "onadata.libs.serializers.user_profile_serializer." + "send_verification_email.delay" + ) ) def test_partial_update_email(self, mock_send_verification_email): profile_data = _profile_data() self._create_user_using_profiles_endpoint(profile_data) rp = RegistrationProfile.objects.get( - user__username=profile_data.get('username') + user__username=profile_data.get("username") ) - deno_extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % rp.user.auth_token - } + deno_extra = {"HTTP_AUTHORIZATION": "Token %s" % rp.user.auth_token} - data = {'email': 'user@example.com', - 'password': "invalid_password"} - request = self.factory.patch('/', data=data, **deno_extra) - response = self.view(request, user=profile_data.get('username')) + data = {"email": "user@example.com", "password": "invalid_password"} + request = self.factory.patch("/", data=data, **deno_extra) + response = self.view(request, user=profile_data.get("username")) self.assertEqual(response.status_code, 400) - data = {'email': 'user@example.com', - 'password': profile_data.get('password')} - request = self.factory.patch('/', data=data, **deno_extra) - response = self.view(request, user=profile_data.get('username')) + data = {"email": "user@example.com", "password": profile_data.get("password")} + request = self.factory.patch("/", data=data, **deno_extra) + response = self.view(request, user=profile_data.get("username")) profile = UserProfile.objects.get(user=rp.user) self.assertEqual(response.status_code, 200) - self.assertEqual(profile.user.email, 'user@example.com') + self.assertEqual(profile.user.email, "user@example.com") rp = RegistrationProfile.objects.get( - user__username=profile_data.get('username') + user__username=profile_data.get("username") ) - self.assertIn('is_email_verified', rp.user.profile.metadata) - self.assertFalse(rp.user.profile.metadata.get('is_email_verified')) + self.assertIn("is_email_verified", rp.user.profile.metadata) + self.assertFalse(rp.user.profile.metadata.get("is_email_verified")) self.assertTrue(mock_send_verification_email.called) def test_update_first_last_name_password_not_affected(self): - data = {'first_name': 'update_first', - 'last_name': 'update_last'} + data = {"first_name": "update_first", "last_name": "update_last"} request = self.factory.patch( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request, user=self.user.username) self.assertEqual(response.status_code, 200) view = ConnectViewSet.as_view( - {'get': 'list'}, - authentication_classes=(DigestAuthentication,)) + {"get": "list"}, authentication_classes=(DigestAuthentication,) + ) - auth = DigestAuth('bob@columbia.edu', 'bobbob') + auth = DigestAuth("bob@columbia.edu", "bobbob") request = self._get_request_session_with_auth(view, auth) response = view(request) self.assertEqual(response.status_code, 200) @patch( - ('onadata.libs.serializers.user_profile_serializer.' - 'send_verification_email.delay') + ( + "onadata.libs.serializers.user_profile_serializer." + "send_verification_email.delay" + ) ) - def test_partial_update_unique_email_api( - self, mock_send_verification_email): + def test_partial_update_unique_email_api(self, mock_send_verification_email): profile_data = { - 'username': u'bobby', - 'first_name': u'Bob', - 'last_name': u'Blender', - 'email': u'bobby@columbia.edu', - 'city': u'Bobville', - 'country': u'US', - 'organization': u'Bob Inc.', - 'website': u'bob.com', - 'twitter': u'boberama', - 'require_auth': False, - 'password': 'bobbob', - 'is_org': False, - 'name': u'Bob Blender' + "username": "bobby", + "first_name": "Bob", + "last_name": "Blender", + "email": "bobby@columbia.edu", + "city": "Bobville", + "country": "US", + "organization": "Bob Inc.", + "website": "bob.com", + "twitter": "boberama", + "require_auth": False, + "password": "bobbob", + "is_org": False, + "name": "Bob Blender", } self._create_user_using_profiles_endpoint(profile_data) rp = RegistrationProfile.objects.get( - user__username=profile_data.get('username') + user__username=profile_data.get("username") ) - deno_extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % rp.user.auth_token - } + deno_extra = {"HTTP_AUTHORIZATION": "Token %s" % rp.user.auth_token} - data = {'email': 'example@gmail.com', - 'password': profile_data.get('password')} + data = {"email": "example@gmail.com", "password": profile_data.get("password")} request = self.factory.patch( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **deno_extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **deno_extra + ) response = self.view(request, user=rp.user.username) self.assertEqual(response.status_code, 200) rp = RegistrationProfile.objects.get( - user__username=profile_data.get('username') + user__username=profile_data.get("username") ) - self.assertIn('is_email_verified', rp.user.profile.metadata) - self.assertFalse(rp.user.profile.metadata.get('is_email_verified')) + self.assertIn("is_email_verified", rp.user.profile.metadata) + self.assertFalse(rp.user.profile.metadata.get("is_email_verified")) self.assertTrue(mock_send_verification_email.called) - self.assertEqual(response.data['email'], data['email']) + self.assertEqual(response.data["email"], data["email"]) # create User request = self.factory.post( - '/api/v1/profiles', data=json.dumps(_profile_data()), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(_profile_data()), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - user = User.objects.get(username='deno') + user = User.objects.get(username="deno") # Update email request = self.factory.patch( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request, user=user.username) self.assertEqual(response.status_code, 400) def test_profile_create_fails_with_long_first_and_last_names(self): data = { - 'username': u'machicimo', - 'email': u'mike@columbia.edu', - 'city': u'Denoville', - 'country': u'US', - 'last_name': - u'undeomnisistenatuserrorsitvoluptatem', - 'first_name': - u'quirationevoluptatemsequinesciunt' + "username": "machicimo", + "email": "mike@columbia.edu", + "city": "Denoville", + "country": "US", + "last_name": "undeomnisistenatuserrorsitvoluptatem", + "first_name": "quirationevoluptatemsequinesciunt", } request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) - self.assertEqual(response.data['first_name'][0], - u'Ensure this field has no more than 30 characters.') - self.assertEqual(response.data['last_name'][0], - u'Ensure this field has no more than 30 characters.') + self.assertEqual( + response.data["first_name"][0], + "Ensure this field has no more than 30 characters.", + ) + self.assertEqual( + response.data["last_name"][0], + "Ensure this field has no more than 30 characters.", + ) self.assertEqual(response.status_code, 400) @all_requests - def grant_perms_form_builder(self, url, request): + def grant_perms_form_builder( + self, url, request + ): # pylint: disable=no-self-use,unused-argument - assert 'Authorization' in request.headers - assert request.headers.get('Authorization').startswith('Token') + assert "Authorization" in request.headers + assert request.headers.get("Authorization").startswith("Token") response = requests.Response() response.status_code = 201 - response._content = \ - { - "detail": "Successfully granted default model level perms to" - " user." - } + setattr( + response, + "_content", + {"detail": "Successfully granted default model level perms to user."}, + ) return response def test_create_user_with_given_name(self): registered_functions = [r[1]() for r in signals.post_save.receivers] self.assertIn(set_kpi_formbuilder_permissions, registered_functions) with HTTMock(self.grant_perms_form_builder): - with self.settings(KPI_FORMBUILDER_URL='http://test_formbuilder$'): + with self.settings(KPI_FORMBUILDER_URL="http://test_formbuilder$"): extra_data = {"username": "rust"} self._login_user_and_profile(extra_post_data=extra_data) @@ -1088,110 +1143,137 @@ def test_get_monthly_submissions(self): """ Test getting monthly submissions for a user """ - view = UserProfileViewSet.as_view({'get': 'monthly_submissions'}) + view = UserProfileViewSet.as_view({"get": "monthly_submissions"}) # publish form and make submissions self._publish_xls_form_to_project() self._make_submissions() count1 = Instance.objects.filter(xform=self.xform).count() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, user=self.user.username) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertFalse(self.xform.shared) - self.assertEquals(response.data, {'private': count1}) + self.assertEqual(response.data, {"private": count1}) # publish another form, make submission and make it public self._publish_form_with_hxl_support() - self.assertEquals(self.xform.id_string, 'hxl_example') - count2 = Instance.objects.filter(xform=self.xform).filter( - date_created__year=datetime.datetime.now().year).filter( - date_created__month=datetime.datetime.now().month).filter( - date_created__day=datetime.datetime.now().day).count() + self.assertEqual(self.xform.id_string, "hxl_example") + count2 = ( + Instance.objects.filter(xform=self.xform) + .filter(date_created__year=datetime.datetime.now().year) + .filter(date_created__month=datetime.datetime.now().month) + .filter(date_created__day=datetime.datetime.now().day) + .count() + ) self.xform.shared = True self.xform.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, user=self.user.username) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data, {'private': count1, 'public': count2}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data, {"private": count1, "public": count2}) def test_get_monthly_submissions_with_year_and_month_params(self): """ Test passing both month and year params """ - view = UserProfileViewSet.as_view({'get': 'monthly_submissions'}) + view = UserProfileViewSet.as_view({"get": "monthly_submissions"}) # publish form and make a submission dated 2013-02-18 self._publish_xls_form_to_project() survey = self.surveys[0] - submission_time = parse_datetime('2013-02-18 15:54:01Z') + submission_time = parse_datetime("2013-02-18 15:54:01Z") self._make_submission( - os.path.join(self.main_directory, 'fixtures', 'transportation', - 'instances', survey, survey + '.xml'), - forced_submission_time=submission_time) - count = Instance.objects.filter(xform=self.xform).filter( - date_created__month=2).filter(date_created__year=2013).count() + os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + survey, + survey + ".xml", + ), + forced_submission_time=submission_time, + ) + count = ( + Instance.objects.filter(xform=self.xform) + .filter(date_created__month=2) + .filter(date_created__year=2013) + .count() + ) # get submission count and assert the response is correct - data = {'month': 2, 'year': 2013} - request = self.factory.get('/', data=data, **self.extra) + data = {"month": 2, "year": 2013} + request = self.factory.get("/", data=data, **self.extra) response = view(request, user=self.user.username) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertFalse(self.xform.shared) - self.assertEquals(response.data, {'private': count}) + self.assertEqual(response.data, {"private": count}) def test_monthly_submissions_with_month_param(self): """ Test that by passing only the value for month, the year is assumed to be the current year """ - view = UserProfileViewSet.as_view({'get': 'monthly_submissions'}) + view = UserProfileViewSet.as_view({"get": "monthly_submissions"}) month = datetime.datetime.now().month year = datetime.datetime.now().year # publish form and make submissions self._publish_xls_form_to_project() self._make_submissions() - count = Instance.objects.filter(xform=self.xform).filter( - date_created__year=year).filter(date_created__month=month).count() + count = ( + Instance.objects.filter(xform=self.xform) + .filter(date_created__year=year) + .filter(date_created__month=month) + .count() + ) - data = {'month': month} - request = self.factory.get('/', data=data, **self.extra) + data = {"month": month} + request = self.factory.get("/", data=data, **self.extra) response = view(request, user=self.user.username) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertFalse(self.xform.shared) - self.assertEquals(response.data, {'private': count}) + self.assertEqual(response.data, {"private": count}) def test_monthly_submissions_with_year_param(self): """ Test that by passing only the value for year the month is assumed to be the current month """ - view = UserProfileViewSet.as_view({'get': 'monthly_submissions'}) + view = UserProfileViewSet.as_view({"get": "monthly_submissions"}) month = datetime.datetime.now().month # publish form and make submissions dated the year 2013 # and the current month self._publish_xls_form_to_project() survey = self.surveys[0] - _time = parse_datetime('2013-' + str(month) + '-18 15:54:01Z') + _time = parse_datetime("2013-" + str(month) + "-18 15:54:01Z") self._make_submission( - os.path.join(self.main_directory, 'fixtures', 'transportation', - 'instances', survey, survey + '.xml'), - forced_submission_time=_time) - count = Instance.objects.filter(xform=self.xform).filter( - date_created__year=2013).filter( - date_created__month=month).count() - - data = {'year': 2013} - request = self.factory.get('/', data=data, **self.extra) + os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + survey, + survey + ".xml", + ), + forced_submission_time=_time, + ) + count = ( + Instance.objects.filter(xform=self.xform) + .filter(date_created__year=2013) + .filter(date_created__month=month) + .count() + ) + + data = {"year": 2013} + request = self.factory.get("/", data=data, **self.extra) response = view(request, user=self.user.username) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertFalse(self.xform.shared) - self.assertEquals(response.data, {'private': count}) + self.assertEqual(response.data, {"private": count}) @override_settings(ENABLE_EMAIL_VERIFICATION=True) - @patch( - 'onadata.apps.api.viewsets.user_profile_viewset.RegistrationProfile') + @patch("onadata.apps.api.viewsets.user_profile_viewset.RegistrationProfile") def test_reads_from_master(self, mock_rp_class): """ Test that on failure to retrieve a UserProfile in the first @@ -1200,88 +1282,92 @@ def test_reads_from_master(self, mock_rp_class): data = _profile_data() self._create_user_using_profiles_endpoint(data) - view = UserProfileViewSet.as_view({'get': 'verify_email'}) - rp = RegistrationProfile.objects.get( - user__username=data.get('username') - ) - _data = {'verification_key': rp.activation_key} + view = UserProfileViewSet.as_view({"get": "verify_email"}) + rp = RegistrationProfile.objects.get(user__username=data.get("username")) + _data = {"verification_key": rp.activation_key} mock_rp_class.DoesNotExist = RegistrationProfile.DoesNotExist mock_rp_class.objects.select_related( - 'user', 'user__profile' - ).get.side_effect = [RegistrationProfile.DoesNotExist, rp] - request = self.factory.get('/', data=_data) + "user", "user__profile" + ).get.side_effect = [RegistrationProfile.DoesNotExist, rp] + request = self.factory.get("/", data=_data) response = view(request) - self.assertEquals(response.status_code, 200) - self.assertIn('is_email_verified', response.data) - self.assertIn('username', response.data) - self.assertTrue(response.data.get('is_email_verified')) - self.assertEquals( - response.data.get('username'), data.get('username')) + self.assertEqual(response.status_code, 200) + self.assertIn("is_email_verified", response.data) + self.assertIn("username", response.data) + self.assertTrue(response.data.get("is_email_verified")) + self.assertEqual(response.data.get("username"), data.get("username")) self.assertEqual( mock_rp_class.objects.select_related( - 'user', 'user__profile').get.call_count, 2) + "user", "user__profile" + ).get.call_count, + 2, + ) def test_change_password_attempts(self): - view = UserProfileViewSet.as_view( - {'post': 'change_password'}) + view = UserProfileViewSet.as_view({"post": "change_password"}) # clear cache - cache.delete('change_password_attempts-bob') - cache.delete('lockout_change_password_user-bob') - self.assertIsNone(cache.get('change_password_attempts-bob')) - self.assertIsNone(cache.get('lockout_change_password_user-bob')) + cache.delete("change_password_attempts-bob") + cache.delete("lockout_change_password_user-bob") + self.assertIsNone(cache.get("change_password_attempts-bob")) + self.assertIsNone(cache.get("lockout_change_password_user-bob")) # first attempt current_password = "wrong_pass" new_password = "bobbob1" - post_data = {'current_password': current_password, - 'new_password': new_password} - request = self.factory.post('/', data=post_data, **self.extra) - response = view(request, user='bob') + post_data = {"current_password": current_password, "new_password": new_password} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request, user="bob") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - "Invalid password." - u" You have 9 attempts left.") - self.assertEqual(cache.get('change_password_attempts-bob'), 1) + self.assertEqual( + response.data, {"error": "Invalid password. You have 9 attempts left."} + ) + self.assertEqual(cache.get("change_password_attempts-bob"), 1) # second attempt - request = self.factory.post('/', data=post_data, **self.extra) - response = view(request, user='bob') + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request, user="bob") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - "Invalid password. You have 8 attempts left.") - self.assertEqual(cache.get('change_password_attempts-bob'), 2) + self.assertEqual( + response.data, {"error": "Invalid password. You have 8 attempts left."} + ) + self.assertEqual(cache.get("change_password_attempts-bob"), 2) # check user is locked out - request = self.factory.post('/', data=post_data, **self.extra) - cache.set('change_password_attempts-bob', 9) - self.assertIsNone(cache.get('lockout_change_password_user-bob')) - response = view(request, user='bob') + request = self.factory.post("/", data=post_data, **self.extra) + cache.set("change_password_attempts-bob", 9) + self.assertIsNone(cache.get("lockout_change_password_user-bob")) + response = view(request, user="bob") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - "Too many password reset attempts," - u" Try again in 30 minutes") - self.assertEqual(cache.get('change_password_attempts-bob'), 10) - self.assertIsNotNone(cache.get('lockout_change_password_user-bob')) + self.assertEqual( + response.data, + {"error": "Too many password reset attempts. Try again in 30 minutes"}, + ) + self.assertEqual(cache.get("change_password_attempts-bob"), 10) + self.assertIsNotNone(cache.get("lockout_change_password_user-bob")) lockout = datetime.datetime.strptime( - cache.get('lockout_change_password_user-bob'), '%Y-%m-%dT%H:%M:%S') + cache.get("lockout_change_password_user-bob"), "%Y-%m-%dT%H:%M:%S" + ) self.assertIsInstance(lockout, datetime.datetime) # clear cache - cache.delete('change_password_attempts-bob') - cache.delete('lockout_change_password_user-bob') + cache.delete("change_password_attempts-bob") + cache.delete("lockout_change_password_user-bob") - @patch('onadata.apps.main.signals.send_generic_email') + @patch("onadata.apps.main.signals.send_generic_email") @override_settings(ENABLE_ACCOUNT_ACTIVATION_EMAILS=True) def test_account_activation_emails(self, mock_send_mail): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() - del data['name'] + del data["name"] request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) @@ -1292,7 +1378,7 @@ def test_account_activation_emails(self, mock_send_mail): mock_send_mail.assert_has_calls( [ call( - data['email'], + data["email"], "\nHi deno,\n\nYour account has been " "successfully created! Kindly wait till an " "administrator activates your account." @@ -1300,14 +1386,14 @@ def test_account_activation_emails(self, mock_send_mail): "Please contact us with any questions, " "we're always happy to help." "\n\nThanks,\nThe Team at Ona", - "Ona account created - Pending activation" + "Ona account created - Pending activation", ), call( - data['email'], + data["email"], "\nHi deno,\n\nYour account has been activated." "\n\nThank you for choosing Ona!" "\n\nThanks,\nThe Team at Ona", - "Ona account activated" - ) + "Ona account activated", + ), ] ) diff --git a/onadata/apps/api/tests/viewsets/test_user_viewset.py b/onadata/apps/api/tests/viewsets/test_user_viewset.py index 7873ff73e0..57461f02ee 100644 --- a/onadata/apps/api/tests/viewsets/test_user_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_user_viewset.py @@ -91,7 +91,7 @@ def test_get_user_using_email(self): request = self.factory.get('/', data=get_params) response = view(request) - self.assertEquals(response.status_code, 401) + self.assertEqual(response.status_code, 401) error = {'detail': 'Authentication credentials were not provided.'} self.assertEqual(response.data, error) @@ -99,7 +99,7 @@ def test_get_user_using_email(self): request = self.factory.get('/', data=get_params, **self.extra) response = view(request) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertEqual(response.data, data) get_params = { @@ -109,7 +109,7 @@ def test_get_user_using_email(self): request = self.factory.get('/', data=get_params, **self.extra) response = view(request) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) # empty results self.assertEqual(response.data, []) @@ -120,7 +120,7 @@ def test_get_user_using_email(self): request = self.factory.get('/', data=get_params, **self.extra) response = view(request) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) # empty results self.assertEqual(response.data, []) @@ -134,15 +134,15 @@ def test_get_non_org_users(self): all_users_request = self.factory.get('/') all_users_response = view(all_users_request) - self.assertEquals(all_users_response.status_code, 200) - self.assertEquals(len( + self.assertEqual(all_users_response.status_code, 200) + self.assertEqual(len( [u for u in all_users_response.data if u['username'] == 'denoinc'] ), 1) no_orgs_request = self.factory.get('/', data={'orgs': 'false'}) no_orgs_response = view(no_orgs_request) - self.assertEquals(no_orgs_response.status_code, 200) - self.assertEquals(len( + self.assertEqual(no_orgs_response.status_code, 200) + self.assertEqual(len( [u for u in no_orgs_response.data if u['username'] == 'denoinc']), 0) diff --git a/onadata/apps/api/tests/viewsets/test_widget_viewset.py b/onadata/apps/api/tests/viewsets/test_widget_viewset.py index 493502c9b6..93ec7cf33a 100644 --- a/onadata/apps/api/tests/viewsets/test_widget_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_widget_viewset.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Test /widgets API endpoint implementation. +""" import os import json @@ -5,51 +9,65 @@ from django.contrib.contenttypes.models import ContentType from onadata.apps.logger.models.widget import Widget -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.viewsets.widget_viewset import WidgetViewSet from onadata.libs.permissions import ReadOnlyRole from onadata.libs.permissions import DataEntryOnlyRole from onadata.libs.permissions import OwnerRole from onadata.apps.api.tools import get_or_create_organization_owners_team -from onadata.apps.api.viewsets.organization_profile_viewset import\ - OrganizationProfileViewSet +from onadata.apps.api.viewsets.organization_profile_viewset import ( + OrganizationProfileViewSet, +) +# pylint: disable=too-many-public-methods class TestWidgetViewSet(TestAbstractViewSet): + """ + Test /widgets API endpoint implementation. + """ + def setUp(self): super(self.__class__, self).setUp() xlsform_path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", "fixtures", - "tutorial.xlsx") + settings.PROJECT_ROOT, "libs", "tests", "utils", "fixtures", "tutorial.xlsx" + ) self._org_create() self._publish_xls_form_to_project(xlsform_path=xlsform_path) for x in range(1, 9): path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - 'tutorial', 'instances', 'uuid{}'.format(x), 'submission.xml') + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial", + "instances", + "uuid{}".format(x), + "submission.xml", + ) self._make_submission(path) x += 1 self._create_dataview() - self.view = WidgetViewSet.as_view({ - 'post': 'create', - 'put': 'update', - 'patch': 'partial_update', - 'delete': 'destroy', - 'get': 'retrieve', - }) + self.view = WidgetViewSet.as_view( + { + "post": "create", + "put": "update", + "patch": "partial_update", + "delete": "destroy", + "get": "retrieve", + } + ) def test_create_widget(self): self._create_widget() def test_create_only_mandatory_fields(self): data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submission_time", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submission_time", } self._create_widget(data) @@ -57,11 +75,11 @@ def test_create_only_mandatory_fields(self): def test_create_using_dataview(self): data = { - 'content_object': 'http://testserver/api/v1/dataviews/%s' % - self.data_view.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submission_time", + "content_object": "http://testserver/api/v1/dataviews/%s" + % self.data_view.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submission_time", } self._create_widget(data) @@ -69,64 +87,61 @@ def test_create_using_dataview(self): def test_create_using_unsupported_model_source(self): data = { - 'content_object': 'http://testserver/api/v1/projects/%s' % - self.project.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submission_time", + "content_object": "http://testserver/api/v1/projects/%s" % self.project.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submission_time", } count = Widget.objects.all().count() - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(count, Widget.objects.all().count()) - self.assertEquals( - response.data['content_object'], - [u"`%s` is not a valid relation." % data['content_object']] + self.assertEqual(response.status_code, 400) + self.assertEqual(count, Widget.objects.all().count()) + self.assertEqual( + response.data["content_object"], + ["`%s` is not a valid relation." % data["content_object"]], ) def test_create_without_required_field(self): data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", } count = Widget.objects.all().count() - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(count, Widget.objects.all().count()) - self.assertEquals(response.data['column'], - [u"This field is required."]) + self.assertEqual(response.status_code, 400) + self.assertEqual(count, Widget.objects.all().count()) + self.assertEqual(response.data["column"], ["This field is required."]) def test_create_unsupported_widget_type(self): data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "table", - 'view_type': "horizontal-bar", - 'column': "_submission_time", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "table", + "view_type": "horizontal-bar", + "column": "_submission_time", } count = Widget.objects.all().count() - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(count, Widget.objects.all().count()) - self.assertEquals(response.data['widget_type'], - [u'"%s" is not a valid choice.' - % data['widget_type']]) + self.assertEqual(response.status_code, 400) + self.assertEqual(count, Widget.objects.all().count()) + self.assertEqual( + response.data["widget_type"], + ['"%s" is not a valid choice.' % data["widget_type"]], + ) def test_update_widget(self): self._create_widget() @@ -134,337 +149,329 @@ def test_update_widget(self): key = self.widget.key data = { - 'title': 'My new title updated', - 'description': 'new description', - 'aggregation': 'new aggregation', - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submission_time", + "title": "My new title updated", + "description": "new description", + "aggregation": "new aggregation", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submission_time", } - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = self.view(request, pk=self.widget.pk) - self.widget = Widget.objects.all().order_by('pk').reverse()[0] + self.widget = Widget.objects.all().order_by("pk").reverse()[0] - self.assertEquals(key, self.widget.key) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['title'], 'My new title updated') - self.assertEquals(response.data['key'], key) - self.assertEquals(response.data['description'], - "new description") - self.assertEquals(response.data['aggregation'], - "new aggregation") + self.assertEqual(key, self.widget.key) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["title"], "My new title updated") + self.assertEqual(response.data["key"], key) + self.assertEqual(response.data["description"], "new description") + self.assertEqual(response.data["aggregation"], "new aggregation") def test_patch_widget(self): self._create_widget() data = { - 'column': "_submitted_by", + "column": "_submitted_by", } - request = self.factory.patch('/', data=data, **self.extra) + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, pk=self.widget.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['column'], '_submitted_by') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["column"], "_submitted_by") def test_delete_widget(self): - ct = ContentType.objects.get(model='xform', app_label='logger') + ct = ContentType.objects.get(model="xform", app_label="logger") self._create_widget() - count = Widget.objects.filter(content_type=ct, - object_id=self.xform.pk).count() + count = Widget.objects.filter(content_type=ct, object_id=self.xform.pk).count() - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = self.view(request, pk=self.widget.pk) - self.assertEquals(response.status_code, 204) + self.assertEqual(response.status_code, 204) - after_count = Widget.objects.filter(content_type=ct, - object_id=self.xform.pk).count() - self.assertEquals(count - 1, after_count) + after_count = Widget.objects.filter( + content_type=ct, object_id=self.xform.pk + ).count() + self.assertEqual(count - 1, after_count) def test_list_widgets(self): self._create_widget() self._publish_xls_form_to_project() data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submitted_by", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submitted_by", } self._create_widget(data=data) - view = WidgetViewSet.as_view({ - 'get': 'list', - }) + view = WidgetViewSet.as_view( + { + "get": "list", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 2) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 2) def test_widget_permission_create(self): - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - view = WidgetViewSet.as_view({ - 'post': 'create' - }) + view = WidgetViewSet.as_view({"post": "create"}) data = { - 'title': "Widget that", - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'description': "Test widget", - 'aggregation': "Sum", - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "age", - 'group_by': '' + "title": "Widget that", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "description": "Test widget", + "aggregation": "Sum", + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "age", + "group_by": "", } # to do: test random user with auth but no perms - request = self.factory.post('/', data=json.dumps(data), - content_type="application/json", - **self.extra) + request = self.factory.post( + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request) - self.assertEquals(response.status_code, 400) + self.assertEqual(response.status_code, 400) # owner OwnerRole.add(self.user, self.project) - request = self.factory.post('/', data=json.dumps(data), - content_type="application/json", - **self.extra) + request = self.factory.post( + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request) - self.assertEquals(response.status_code, 201) + self.assertEqual(response.status_code, 201) # readonly ReadOnlyRole.add(self.user, self.project) - request = self.factory.post('/', data=json.dumps(data), - content_type="application/json", - **self.extra) + request = self.factory.post( + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request) - self.assertEquals(response.status_code, 201) + self.assertEqual(response.status_code, 201) # dataentryonlyrole DataEntryOnlyRole.add(self.user, self.project) - request = self.factory.post('/', data=json.dumps(data), - content_type="application/json", - **self.extra) + request = self.factory.post( + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request) - self.assertEquals(response.status_code, 201) + self.assertEqual(response.status_code, 201) def test_widget_permission_change(self): self._create_widget() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) data = { - 'title': "Widget those", + "title": "Widget those", } OwnerRole.add(self.user, self.project) OwnerRole.add(self.user, self.xform) - request = self.factory.patch('/', data=data, **self.extra) + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, pk=self.widget.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['title'], 'Widget those') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["title"], "Widget those") ReadOnlyRole.add(self.user, self.project) ReadOnlyRole.add(self.user, self.xform) - request = self.factory.patch('/', data=data, **self.extra) + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, pk=self.widget.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['title'], 'Widget those') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["title"], "Widget those") def test_widget_permission_list(self): self._create_widget() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - view = WidgetViewSet.as_view({ - 'get': 'list', - }) + view = WidgetViewSet.as_view( + { + "get": "list", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 0) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 0) # assign alice the perms ReadOnlyRole.add(self.user, self.xform) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 1) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 1) def test_widget_permission_get(self): self._create_widget() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.widget.pk) - self.assertEquals(response.status_code, 404) + self.assertEqual(response.status_code, 404) # assign alice the perms ReadOnlyRole.add(self.user, self.project) - request = self.factory.get('/', **self.extra) - response = self.view(request, formid=self.xform.pk, - pk=self.widget.pk) + request = self.factory.get("/", **self.extra) + response = self.view(request, formid=self.xform.pk, pk=self.widget.pk) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) def test_widget_data(self): self._create_widget() - data = { - "data": True - } + data = {"data": True} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = self.view(request, pk=self.widget.pk) self.assertEqual(response.status_code, 200) - self.assertIsNotNone(response.data.get('data')) - self.assertIn('data', response.data.get('data')) - self.assertEquals(len(response.data.get('data')['data']), 8) - self.assertIn('age', response.data.get('data')['data'][0]) - self.assertIn('count', response.data.get('data')['data'][0]) - - self.assertEqual(response.data.get('data')['data_type'], 'numeric') - self.assertEqual(response.data.get('data')['field_label'], - 'How old are you?') - self.assertEqual(response.data.get('data')['field_type'], 'integer') - self.assertEqual(response.data.get('data')['field_xpath'], 'age') + self.assertIsNotNone(response.data.get("data")) + self.assertIn("data", response.data.get("data")) + self.assertEqual(len(response.data.get("data")["data"]), 8) + self.assertIn("age", response.data.get("data")["data"][0]) + self.assertIn("count", response.data.get("data")["data"][0]) + + self.assertEqual(response.data.get("data")["data_type"], "numeric") + self.assertEqual(response.data.get("data")["field_label"], "How old are you?") + self.assertEqual(response.data.get("data")["field_type"], "integer") + self.assertEqual(response.data.get("data")["field_xpath"], "age") def test_widget_data_with_group_by(self): - self._create_widget(group_by='gender') + self._create_widget(group_by="gender") - data = { - "data": True - } + data = {"data": True} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = self.view(request, pk=self.widget.pk) self.assertEqual(response.status_code, 200) - self.assertIsNotNone(response.data.get('data')) - self.assertIn('data', response.data.get('data')) - self.assertEquals(len(response.data.get('data')['data']), 2) - self.assertIn('gender', response.data.get('data')['data'][0]) - self.assertIn('sum', response.data.get('data')['data'][0]) - self.assertIn('mean', response.data.get('data')['data'][0]) + self.assertIsNotNone(response.data.get("data")) + self.assertIn("data", response.data.get("data")) + self.assertEqual(len(response.data.get("data")["data"]), 2) + self.assertIn("gender", response.data.get("data")["data"][0]) + self.assertIn("sum", response.data.get("data")["data"][0]) + self.assertIn("mean", response.data.get("data")["data"][0]) def test_widget_data_widget(self): data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "gender", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "gender", } self._create_widget(data) - data = { - "data": True - } - request = self.factory.get('/', data=data, **self.extra) + data = {"data": True} + request = self.factory.get("/", data=data, **self.extra) response = self.view(request, pk=self.widget.pk) self.assertEqual(response.status_code, 200) - self.assertIsNotNone(response.data.get('data')) - self.assertEquals(response.data.get('data'), - { - 'field_type': u'select one', - 'data_type': 'categorized', - 'field_xpath': u'gender', - 'field_label': u'Gender', - 'grouped_by': None, - 'data': [ - {'count': 1, 'gender': u'female'}, - {'count': 7, 'gender': u'male'}]}) + self.assertIsNotNone(response.data.get("data")) + self.assertEqual( + response.data.get("data"), + { + "field_type": "select one", + "data_type": "categorized", + "field_xpath": "gender", + "field_label": "Gender", + "grouped_by": None, + "data": [ + {"count": 1, "gender": "female"}, + {"count": 7, "gender": "male"}, + ], + }, + ) def test_widget_with_key(self): self._create_widget() - view = WidgetViewSet.as_view({ - 'get': 'list', - }) + view = WidgetViewSet.as_view( + { + "get": "list", + } + ) - data = { - "key": self.widget.key - } + data = {"key": self.widget.key} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, formid=self.xform.pk) self.assertEqual(response.status_code, 200) - self.assertIsNotNone(response.data.get('data')) - self.assertIn('data', response.data.get('data')) - self.assertEquals(len(response.data.get('data')['data']), 8) - self.assertIn('age', response.data.get('data')['data'][0]) - self.assertIn('count', response.data.get('data')['data'][0]) + self.assertIsNotNone(response.data.get("data")) + self.assertIn("data", response.data.get("data")) + self.assertEqual(len(response.data.get("data")["data"]), 8) + self.assertIn("age", response.data.get("data")["data"][0]) + self.assertIn("count", response.data.get("data")["data"][0]) def test_widget_with_key_anon(self): self._create_widget() - view = WidgetViewSet.as_view({ - 'get': 'list', - }) + view = WidgetViewSet.as_view( + { + "get": "list", + } + ) - data = { - "key": self.widget.key - } + data = {"key": self.widget.key} # Anonymous user can access the widget self.extra = {} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, formid=self.xform.pk) self.assertEqual(response.status_code, 200) - self.assertIsNotNone(response.data.get('data')) - self.assertIn('data', response.data.get('data')) - self.assertEquals(len(response.data.get('data')['data']), 8) - self.assertIn('age', response.data.get('data')['data'][0]) - self.assertIn('count', response.data.get('data')['data'][0]) + self.assertIsNotNone(response.data.get("data")) + self.assertIn("data", response.data.get("data")) + self.assertEqual(len(response.data.get("data")["data"]), 8) + self.assertIn("age", response.data.get("data")["data"][0]) + self.assertIn("count", response.data.get("data")["data"][0]) def test_widget_with_nonexistance_key(self): self._create_widget() - view = WidgetViewSet.as_view({ - 'get': 'list', - }) + view = WidgetViewSet.as_view( + { + "get": "list", + } + ) - data = { - "key": "randomkeythatdoesnotexist" - } + data = {"key": "randomkeythatdoesnotexist"} self.extra = {} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=self.xform.pk) self.assertEqual(response.status_code, 404) @@ -472,246 +479,247 @@ def test_widget_with_nonexistance_key(self): def test_widget_data_public_form(self): self._create_widget() - view = WidgetViewSet.as_view({ - 'get': 'list', - }) + view = WidgetViewSet.as_view( + { + "get": "list", + } + ) self.extra = {} - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) - self.assertEquals(len(response.data), 0) + self.assertEqual(len(response.data), 0) # Anonymous user can access widget in public form self.xform.shared_data = True self.xform.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, formid=self.xform.pk) self.assertEqual(response.status_code, 200) - self.assertEquals(len(response.data), 1) + self.assertEqual(len(response.data), 1) def test_widget_pk_formid_required(self): self._create_widget() data = { - 'title': 'My new title updated', - 'description': 'new description', - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submission_time", + "title": "My new title updated", + "description": "new description", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submission_time", } - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(response.data, - {u'detail': u"'pk' required for this" - u" action"}) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.data, {"detail": "'pk' required for this" " action"}) def test_list_widgets_with_formid(self): self._create_widget() self._publish_xls_form_to_project() data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submitted_by", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submitted_by", } self._create_widget(data=data) - view = WidgetViewSet.as_view({ - 'get': 'list', - }) + view = WidgetViewSet.as_view( + { + "get": "list", + } + ) - data = { - "xform": self.xform.pk - } + data = {"xform": self.xform.pk} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 1) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 1) def test_create_column_not_in_form(self): data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "doesnotexists", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "doesnotexists", } count = Widget.objects.all().count() - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(count, Widget.objects.all().count()) - self.assertEquals(response.data['column'], - [u"'doesnotexists' not in the form."]) + self.assertEqual(response.status_code, 400) + self.assertEqual(count, Widget.objects.all().count()) + self.assertEqual(response.data["column"], ["'doesnotexists' not in the form."]) def test_create_widget_with_xform_no_perms(self): data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "age", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "age", } - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(response.data['content_object'], - [u"You don't have permission to the Project."]) + self.assertEqual(response.status_code, 400) + self.assertEqual( + response.data["content_object"], + ["You don't have permission to the Project."], + ) def test_filter_widgets_by_dataview(self): self._create_widget() self._publish_xls_form_to_project() data = { - 'content_object': 'http://testserver/api/v1/dataviews/%s' % - self.data_view.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submitted_by", + "content_object": "http://testserver/api/v1/dataviews/%s" + % self.data_view.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submitted_by", } self._create_widget(data=data) data = { - 'content_object': 'http://testserver/api/v1/dataviews/%s' % - self.data_view.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submission_time", + "content_object": "http://testserver/api/v1/dataviews/%s" + % self.data_view.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submission_time", } self._create_widget(data) - view = WidgetViewSet.as_view({ - 'get': 'list', - }) + view = WidgetViewSet.as_view( + { + "get": "list", + } + ) - data = { - "dataview": self.data_view.pk - } + data = {"dataview": self.data_view.pk} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 2) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 2) - data = { - "dataview": "so_invalid" - } + data = {"dataview": "so_invalid"} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(response.data['detail'], - u"Invalid value for dataview %s." % "so_invalid") + self.assertEqual(response.status_code, 400) + self.assertEqual( + response.data["detail"], "Invalid value for dataview %s." % "so_invalid" + ) def test_order_widget(self): self._create_widget() self._create_widget() self._create_widget() - data = { - 'column': "_submission_time", - 'order': 1 - } + data = {"column": "_submission_time", "order": 1} - request = self.factory.patch('/', data=data, **self.extra) + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, pk=self.widget.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['order'], 1) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["order"], 1) - widget = Widget.objects.all().order_by('pk')[0] - self.assertEquals(widget.order, 0) + widget = Widget.objects.all().order_by("pk")[0] + self.assertEqual(widget.order, 0) - widget = Widget.objects.all().order_by('pk')[1] - self.assertEquals(widget.order, 2) + widget = Widget.objects.all().order_by("pk")[1] + self.assertEqual(widget.order, 2) def test_widget_data_case_sensitive(self): xlsform_path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", "fixtures", - "tutorial_2.xlsx") + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial_2.xlsx", + ) self._publish_xls_form_to_project(xlsform_path=xlsform_path) for x in range(1, 9): path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - 'tutorial_2', 'instances', 'uuid{}'.format(x), - 'submission.xml') + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial_2", + "instances", + "uuid{}".format(x), + "submission.xml", + ) self._make_submission(path) x += 1 data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "Gender", - 'metadata': { - "test metadata": "percentage" - } + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "Gender", + "metadata": {"test metadata": "percentage"}, } self._create_widget(data) - data = { - "data": True - } - request = self.factory.get('/', data=data, **self.extra) + data = {"data": True} + request = self.factory.get("/", data=data, **self.extra) response = self.view(request, pk=self.widget.pk) self.assertEqual(response.status_code, 200) - self.assertIsNotNone(response.data.get('data')) - self.assertEquals(response.data.get('data'), - { - 'field_type': u'select one', - 'data_type': 'categorized', - 'field_xpath': u'Gender', - 'field_label': u'Gender', - 'grouped_by': None, - 'data': [ - {'count': 1, 'Gender': u'female'}, - {'count': 7, 'Gender': u'male'}]}) + self.assertIsNotNone(response.data.get("data")) + self.assertEqual( + response.data.get("data"), + { + "field_type": "select one", + "data_type": "categorized", + "field_xpath": "Gender", + "field_label": "Gender", + "grouped_by": None, + "data": [ + {"count": 1, "Gender": "female"}, + {"count": 7, "Gender": "male"}, + ], + }, + ) def test_widget_create_by_org_admin(self): self.project.organization = self.organization.user self.project.save() - chuck_data = {'username': 'chuck', 'email': 'chuck@localhost.com'} + chuck_data = {"username": "chuck", "email": "chuck@localhost.com"} chuck_profile = self._create_user_profile(chuck_data) - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) + view = OrganizationProfileViewSet.as_view({"post": "members"}) - data = {'username': chuck_profile.user.username, - 'role': OwnerRole.name} + data = {"username": chuck_profile.user.username, "role": OwnerRole.name} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, user=self.organization.user.username) @@ -720,88 +728,78 @@ def test_widget_create_by_org_admin(self): owners_team = get_or_create_organization_owners_team(self.organization) self.assertIn(chuck_profile.user, owners_team.user_set.all()) - extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % chuck_profile.user.auth_token} + extra = {"HTTP_AUTHORIZATION": "Token %s" % chuck_profile.user.auth_token} - view = WidgetViewSet.as_view({ - 'post': 'create' - }) + view = WidgetViewSet.as_view({"post": "create"}) data = { - 'content_object': 'http://testserver/api/v1/dataviews/%s' % - self.data_view.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submission_time", + "content_object": "http://testserver/api/v1/dataviews/%s" + % self.data_view.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submission_time", } - request = self.factory.post('/', data=json.dumps(data), - content_type="application/json", - **extra) + request = self.factory.post( + "/", data=json.dumps(data), content_type="application/json", **extra + ) response = view(request) - self.assertEquals(response.status_code, 201) + self.assertEqual(response.status_code, 201) def test_create_multiple_choice(self): data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "favorite_toppings/pepperoni", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "favorite_toppings/pepperoni", } self._create_widget(data) - data = { - "data": True - } + data = {"data": True} pk = self.widget.pk - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = self.view(request, pk=pk) self.assertEqual(response.status_code, 200) form_pk = self.xform.pk expected = { - 'content_object': - u'http://testserver/api/v1/forms/{}'.format(form_pk), - 'description': None, - 'title': None, - 'url': u'http://testserver/api/v1/widgets/{}'.format(pk), - 'view_type': u'horizontal-bar', - 'aggregation': None, - 'order': 0, - 'widget_type': 'charts', - 'column': u'favorite_toppings/pepperoni', - 'group_by': None, - 'key': self.widget.key, - 'data': { - 'field_type': u'', - 'data_type': 'categorized', - 'field_xpath': u'favorite_toppings/pepperoni', - 'grouped_by': None, - 'field_label': u'Pepperoni', - 'data': [{'' - 'count': 0, - 'favorite_toppings/pepperoni': None - }] - }, - 'id': pk, - 'metadata': {} + "content_object": "http://testserver/api/v1/forms/{}".format(form_pk), + "description": None, + "title": None, + "url": "http://testserver/api/v1/widgets/{}".format(pk), + "view_type": "horizontal-bar", + "aggregation": None, + "order": 0, + "widget_type": "charts", + "column": "favorite_toppings/pepperoni", + "group_by": None, + "key": self.widget.key, + "data": { + "field_type": "", + "data_type": "categorized", + "field_xpath": "favorite_toppings/pepperoni", + "grouped_by": None, + "field_label": "Pepperoni", + "data": [{"" "count": 0, "favorite_toppings/pepperoni": None}], + }, + "id": pk, + "metadata": {}, } self.assertEqual(expected, response.data) def test_create_long_title(self): data = { - 'title': 'When editing grouped charts titles, much as the title ' - 'can be edited, it cant be saved as the title exceeds 50', - 'content_object': 'http://testserver/api/v1/dataviews/%s' % - self.data_view.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submission_time", + "title": "When editing grouped charts titles, much as the title " + "can be edited, it cant be saved as the title exceeds 50", + "content_object": "http://testserver/api/v1/dataviews/%s" + % self.data_view.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submission_time", } self._create_widget(data) diff --git a/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py index c6dcb9d915..66b962f63a 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py @@ -15,10 +15,11 @@ import simplejson as json from django_digest.test import DigestAuth -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet -from onadata.apps.api.viewsets.xform_submission_viewset import \ - XFormSubmissionViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import ( + TestAbstractViewSet, + add_uuid_to_submission_xml, +) +from onadata.apps.api.viewsets.xform_submission_viewset import XFormSubmissionViewSet from onadata.apps.logger.models import Attachment, Instance, XForm from onadata.libs.permissions import DataEntryRole from onadata.libs.utils.common_tools import get_uuid @@ -29,12 +30,10 @@ class TestXFormSubmissionViewSet(TestAbstractViewSet, TransactionTestCase): """ TestXFormSubmissionViewSet test class. """ + def setUp(self): super(TestXFormSubmissionViewSet, self).setUp() - self.view = XFormSubmissionViewSet.as_view({ - "head": "create", - "post": "create" - }) + self.view = XFormSubmissionViewSet.as_view({"head": "create", "post": "create"}) self._publish_xls_form_to_project() def test_unique_instanceid_per_form_only(self): @@ -42,11 +41,10 @@ def test_unique_instanceid_per_form_only(self): Test unique instanceID submissions per form. """ self._make_submissions() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice = self._create_user_profile(alice_data) self.user = alice.user - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} self._publish_xls_form_to_project() self._make_submissions() @@ -56,31 +54,40 @@ def test_post_submission_anonymous(self): """ s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - with open(submission_path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} - request = self.factory.post( - '/%s/submission' % self.user.username, data) + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + with open(submission_path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} + request = self.factory.post("/%s/submission" % self.user.username, data) request.user = AnonymousUser() response = self.view(request, username=self.user.username) - self.assertContains(response, 'Successful submission', - status_code=201) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'text/xml; charset=utf-8') - self.assertEqual(response['Location'], - 'http://testserver/%s/submission' - % self.user.username) + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "text/xml; charset=utf-8") + self.assertEqual( + response["Location"], + "http://testserver/%s/submission" % self.user.username, + ) def test_username_case_insensitive(self): """ @@ -88,39 +95,47 @@ def test_username_case_insensitive(self): to the username case """ # Change username to Bob - self.user.username = 'Bob' + self.user.username = "Bob" self.user.save() survey = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', survey, media_file) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + survey, + media_file, + ) form = XForm.objects.get(user=self.user) count = form.submission_count() - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', survey, survey + '.xml') - with open(submission_path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} + self.main_directory, + "fixtures", + "transportation", + "instances", + survey, + survey + ".xml", + ) + with open(submission_path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} # Make submission to /bob/submission - request = self.factory.post( - '/%s/submission' % 'bob', data) + request = self.factory.post("/%s/submission" % "bob", data) request.user = AnonymousUser() response = self.view(request, username=self.user.username) # Submission should be submitted to the right form - self.assertContains(response, 'Successful submission', - status_code=201) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'text/xml; charset=utf-8') - self.assertEqual(count+1, form.submission_count()) + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "text/xml; charset=utf-8") + self.assertEqual(count + 1, form.submission_count()) def test_post_submission_authenticated(self): """ @@ -128,58 +143,78 @@ def test_post_submission_authenticated(self): """ s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - with open(submission_path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} - request = self.factory.post('/submission', data) + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + with open(submission_path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} + request = self.factory.post("/submission", data) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request, username=self.user.username) - self.assertContains(response, 'Successful submission', - status_code=201) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'text/xml; charset=utf-8') - self.assertEqual(response['Location'], - 'http://testserver/submission') + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "text/xml; charset=utf-8") + self.assertEqual(response["Location"], "http://testserver/submission") def test_post_submission_uuid_other_user_username_not_provided(self): """ Test submission without formhub/uuid done by a different user who has no permission to the form fails. """ - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._create_user_profile(alice_data) s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - path = self._add_uuid_to_submission_xml(path, self.xform) + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + path = add_uuid_to_submission_xml(path, self.xform) - with open(path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} - request = self.factory.post('/submission', data) + with open(path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} + request = self.factory.post("/submission", data) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('alice', 'bobbob') + auth = DigestAuth("alice", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request) self.assertEqual(response.status_code, 404) @@ -190,23 +225,34 @@ def test_post_submission_uuid_duplicate_no_username_provided(self): from the original is properly routed to the request users version of the form """ - alice_profile = self._create_user_profile({ - 'username': 'alice', - 'email': 'alice@localhost.com' - }) + alice_profile = self._create_user_profile( + {"username": "alice", "email": "alice@localhost.com"} + ) s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - path = self._add_uuid_to_submission_xml(path, self.xform) + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + path = add_uuid_to_submission_xml(path, self.xform) - with open(path, 'rb') as sf: + with open(path, "rb") as sf: # Submits to the correct form count = XForm.objects.count() original_form_pk = self.xform.pk @@ -221,21 +267,19 @@ def test_post_submission_uuid_duplicate_no_username_provided(self): self.assertEqual(XForm.objects.count(), count + 1) request = self.factory.post( - '/submission', - {'xml_submission_file': sf, 'media_file': f}) + "/submission", {"xml_submission_file": sf, "media_file": f} + ) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('alice', 'bobbob') + auth = DigestAuth("alice", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request) self.assertEqual(response.status_code, 201) duplicate_form.refresh_from_db() + self.assertEqual(duplicate_form.instances.all().count(), 1) self.assertEqual( - duplicate_form.instances.all().count(), - 1) - self.assertEqual( - XForm.objects.get( - pk=original_form_pk).instances.all().count(), 0) + XForm.objects.get(pk=original_form_pk).instances.all().count(), 0 + ) def test_post_submission_authenticated_json(self): """ @@ -243,28 +287,25 @@ def test_post_submission_authenticated_json(self): """ path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - '..', - 'fixtures', - 'transport_submission.json') - with open(path, encoding='utf-8') as f: + "..", + "fixtures", + "transport_submission.json", + ) + with open(path, encoding="utf-8") as f: data = json.loads(f.read()) - request = self.factory.post('/submission', data, format='json') + request = self.factory.post("/submission", data, format="json") response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request) - self.assertContains(response, 'Successful submission', - status_code=201) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'application/json') - self.assertEqual(response['Location'], - 'http://testserver/submission') + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "application/json") + self.assertEqual(response["Location"], "http://testserver/submission") def test_post_submission_authenticated_bad_json_list(self): """ @@ -273,28 +314,25 @@ def test_post_submission_authenticated_bad_json_list(self): """ path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - '..', - 'fixtures', - 'transport_submission.json') - with open(path, encoding='utf-8') as f: + "..", + "fixtures", + "transport_submission.json", + ) + with open(path, encoding="utf-8") as f: data = json.loads(f.read()) - request = self.factory.post('/submission', [data], format='json') + request = self.factory.post("/submission", [data], format="json") response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request) - self.assertContains(response, 'Invalid data', - status_code=400) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'application/json') - self.assertEqual(response['Location'], - 'http://testserver/submission') + self.assertContains(response, "Invalid data", status_code=400) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "application/json") + self.assertEqual(response["Location"], "http://testserver/submission") def test_post_submission_authenticated_bad_json_submission_list(self): """ @@ -303,29 +341,26 @@ def test_post_submission_authenticated_bad_json_submission_list(self): """ path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - '..', - 'fixtures', - 'transport_submission.json') - with open(path, encoding='utf-8') as f: + "..", + "fixtures", + "transport_submission.json", + ) + with open(path, encoding="utf-8") as f: data = json.loads(f.read()) - data['submission'] = [data['submission']] - request = self.factory.post('/submission', data, format='json') + data["submission"] = [data["submission"]] + request = self.factory.post("/submission", data, format="json") response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request) - self.assertContains(response, 'Incorrect format', - status_code=400) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'application/json') - self.assertEqual(response['Location'], - 'http://testserver/submission') + self.assertContains(response, "Incorrect format", status_code=400) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "application/json") + self.assertEqual(response["Location"], "http://testserver/submission") def test_post_submission_authenticated_bad_json(self): """ @@ -333,29 +368,26 @@ def test_post_submission_authenticated_bad_json(self): """ path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - '..', - 'fixtures', - 'transport_submission_bad.json') - with open(path, encoding='utf-8') as f: + "..", + "fixtures", + "transport_submission_bad.json", + ) + with open(path, encoding="utf-8") as f: data = json.loads(f.read()) - request = self.factory.post('/submission', data, format='json') + request = self.factory.post("/submission", data, format="json") response = self.view(request) self.assertEqual(response.status_code, 401) - request = self.factory.post('/submission', data, format='json') - auth = DigestAuth('bob', 'bobbob') + request = self.factory.post("/submission", data, format="json") + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request) - self.assertContains(response, 'Received empty submission', - status_code=400) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'application/json') - self.assertEqual(response['Location'], - 'http://testserver/submission') + self.assertContains(response, "Received empty submission", status_code=400) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "application/json") + self.assertEqual(response["Location"], "http://testserver/submission") def test_post_submission_require_auth(self): """ @@ -365,28 +397,29 @@ def test_post_submission_require_auth(self): self.user.profile.save() submission = self.surveys[0] submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', submission, submission + '.xml') - with open(submission_path, 'rb') as submission_file: - data = {'xml_submission_file': submission_file} - request = self.factory.post('/submission', data) + self.main_directory, + "fixtures", + "transportation", + "instances", + submission, + submission + ".xml", + ) + with open(submission_path, "rb") as submission_file: + data = {"xml_submission_file": submission_file} + request = self.factory.post("/submission", data) response = self.view(request) self.assertEqual(response.status_code, 401) response = self.view(request, username=self.user.username) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request, username=self.user.username) - self.assertContains(response, 'Successful submission', - status_code=201) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'text/xml; charset=utf-8') - self.assertEqual(response['Location'], - 'http://testserver/submission') + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "text/xml; charset=utf-8") + self.assertEqual(response["Location"], "http://testserver/submission") def test_post_submission_require_auth_anonymous_user(self): """ @@ -398,17 +431,29 @@ def test_post_submission_require_auth_anonymous_user(self): count = Attachment.objects.count() s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - with open(submission_path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} - request = self.factory.post('/submission', data) + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + with open(submission_path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} + request = self.factory.post("/submission", data) response = self.view(request) self.assertEqual(response.status_code, 401) response = self.view(request, username=self.user.username) @@ -423,35 +468,48 @@ def test_post_submission_require_auth_other_user(self): self.user.profile.require_auth = True self.user.profile.save() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._create_user_profile(alice_data) count = Attachment.objects.count() s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - with open(submission_path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} - request = self.factory.post('/submission', data) + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + with open(submission_path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} + request = self.factory.post("/submission", data) response = self.view(request) self.assertEqual(response.status_code, 401) response = self.view(request, username=self.user.username) self.assertEqual(response.status_code, 401) self.assertEqual(count, Attachment.objects.count()) - auth = DigestAuth('alice', 'bobbob') + auth = DigestAuth("alice", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request, username=self.user.username) self.assertContains( response, - 'alice is not allowed to make submissions to bob', - status_code=403) + "alice is not allowed to make submissions to bob", + status_code=403, + ) def test_post_submission_require_auth_data_entry_role(self): """ @@ -461,63 +519,81 @@ def test_post_submission_require_auth_data_entry_role(self): self.user.profile.require_auth = True self.user.profile.save() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) DataEntryRole.add(alice_profile.user, self.xform) count = Attachment.objects.count() s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - with open(submission_path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} - request = self.factory.post('/submission', data) + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + with open(submission_path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} + request = self.factory.post("/submission", data) response = self.view(request) self.assertEqual(response.status_code, 401) response = self.view(request, username=self.user.username) self.assertEqual(response.status_code, 401) self.assertEqual(count, Attachment.objects.count()) - auth = DigestAuth('alice', 'bobbob') + auth = DigestAuth("alice", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request, username=self.user.username) - self.assertContains(response, 'Successful submission', - status_code=201) + self.assertContains(response, "Successful submission", status_code=201) def test_post_submission_json_without_submission_key(self): """ Tesut JSON submission without the submission key fails. """ data = {"id": "transportation_2011_07_25"} - request = self.factory.post('/submission', data, format='json') + request = self.factory.post("/submission", data, format="json") response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request) - self.assertContains(response, 'No submission key provided.', - status_code=400) + self.assertContains(response, "No submission key provided.", status_code=400) def test_NaN_in_submission(self): """ Test submissions with uuid as NaN are successful. """ xlsform_path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", "fixtures", - "tutorial.xlsx") + settings.PROJECT_ROOT, "libs", "tests", "utils", "fixtures", "tutorial.xlsx" + ) self._publish_xls_form_to_project(xlsform_path=xlsform_path) path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - 'tutorial', 'instances', 'uuid_NaN', 'submission.xml') + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial", + "instances", + "uuid_NaN", + "submission.xml", + ) self._make_submission(path) def test_rapidpro_post_submission(self): @@ -525,112 +601,112 @@ def test_rapidpro_post_submission(self): Test a Rapidpro Webhook POST submission. """ # pylint: disable=C0301 - data = json.dumps({ - 'contact': { - 'name': 'Davis', - 'urn': 'tel:+12065551212', - 'uuid': '23dae14f-7202-4ff5-bdb6-2390d2769968' - }, - 'flow': { - 'name': '1166', - 'uuid': '9da5e439-35af-4ecb-b7fc-2911659f6b04' - }, - 'results': { - 'fruit_name': { - 'category': 'All Responses', - 'value': 'orange' + data = json.dumps( + { + "contact": { + "name": "Davis", + "urn": "tel:+12065551212", + "uuid": "23dae14f-7202-4ff5-bdb6-2390d2769968", + }, + "flow": { + "name": "1166", + "uuid": "9da5e439-35af-4ecb-b7fc-2911659f6b04", + }, + "results": { + "fruit_name": {"category": "All Responses", "value": "orange"}, }, } - }) + ) request = self.factory.post( - '/submission', data, - content_type='application/json', - HTTP_USER_AGENT='RapidProMailroom/5.2.0') + "/submission", + data, + content_type="application/json", + HTTP_USER_AGENT="RapidProMailroom/5.2.0", + ) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) - response = self.view(request, username=self.user.username, - xform_pk=self.xform.pk) - self.assertContains(response, 'Successful submission', status_code=201) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Location'], 'http://testserver/submission') + response = self.view( + request, username=self.user.username, xform_pk=self.xform.pk + ) + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Location"], "http://testserver/submission") # InstanceID is returned as uuid:. # Retrieving the uuid without the prefix in order to retrieve # the Instance object - uuid = response.data.get('instanceID').split(':')[1] + uuid = response.data.get("instanceID").split(":")[1] instance = Instance.objects.get(uuid=uuid) expected_xml = ( f"<{self.xform.survey.name} id=" "'transportation_2011_07_25'>orange" f"\n\n uuid:{uuid}" - f"\n") + f"\n" + ) self.assertEqual(instance.xml, expected_xml) - self.assertEqual(self.xform.survey.name, 'data') + self.assertEqual(self.xform.survey.name, "data") def test_legacy_rapidpro_post_submission(self): """ Test a Legacy Rapidpro Webhook POST submission. """ # pylint: disable=C0301 - data = 'run=76250&text=orange&flow_uuid=9da5e439-35af-4ecb-b7fc-2911659f6b04&phone=%2B12065550100&step=3b15df81-a0bd-4de7-8186-145ea3bb6c43&contact_name=Antonate+Maritim&flow_name=fruit&header=Authorization&urn=tel%3A%2B12065550100&flow=1166&relayer=-1&contact=fe4df540-39c1-4647-b4bc-1c93833e22e0&values=%5B%7B%22category%22%3A+%7B%22base%22%3A+%22All+Responses%22%7D%2C+%22node%22%3A+%228037c12f-a277-4255-b630-6a03b035767a%22%2C+%22time%22%3A+%222017-10-04T07%3A18%3A08.171069Z%22%2C+%22text%22%3A+%22orange%22%2C+%22rule_value%22%3A+%22orange%22%2C+%22value%22%3A+%22orange%22%2C+%22label%22%3A+%22fruit_name%22%7D%5D&time=2017-10-04T07%3A18%3A08.205524Z&steps=%5B%7B%22node%22%3A+%220e18202f-9ec4-4756-b15b-e9f152122250%22%2C+%22arrived_on%22%3A+%222017-10-04T07%3A15%3A17.548657Z%22%2C+%22left_on%22%3A+%222017-10-04T07%3A15%3A17.604668Z%22%2C+%22text%22%3A+%22Fruit%3F%22%2C+%22type%22%3A+%22A%22%2C+%22value%22%3A+null%7D%2C+%7B%22node%22%3A+%228037c12f-a277-4255-b630-6a03b035767a%22%2C+%22arrived_on%22%3A+%222017-10-04T07%3A15%3A17.604668Z%22%2C+%22left_on%22%3A+%222017-10-04T07%3A18%3A08.171069Z%22%2C+%22text%22%3A+%22orange%22%2C+%22type%22%3A+%22R%22%2C+%22value%22%3A+%22orange%22%7D%2C+%7B%22node%22%3A+%223b15df81-a0bd-4de7-8186-145ea3bb6c43%22%2C+%22arrived_on%22%3A+%222017-10-04T07%3A18%3A08.171069Z%22%2C+%22left_on%22%3A+null%2C+%22text%22%3A+null%2C+%22type%22%3A+%22A%22%2C+%22value%22%3A+null%7D%5D&flow_base_language=base&channel=-1' # noqa + data = "run=76250&text=orange&flow_uuid=9da5e439-35af-4ecb-b7fc-2911659f6b04&phone=%2B12065550100&step=3b15df81-a0bd-4de7-8186-145ea3bb6c43&contact_name=Antonate+Maritim&flow_name=fruit&header=Authorization&urn=tel%3A%2B12065550100&flow=1166&relayer=-1&contact=fe4df540-39c1-4647-b4bc-1c93833e22e0&values=%5B%7B%22category%22%3A+%7B%22base%22%3A+%22All+Responses%22%7D%2C+%22node%22%3A+%228037c12f-a277-4255-b630-6a03b035767a%22%2C+%22time%22%3A+%222017-10-04T07%3A18%3A08.171069Z%22%2C+%22text%22%3A+%22orange%22%2C+%22rule_value%22%3A+%22orange%22%2C+%22value%22%3A+%22orange%22%2C+%22label%22%3A+%22fruit_name%22%7D%5D&time=2017-10-04T07%3A18%3A08.205524Z&steps=%5B%7B%22node%22%3A+%220e18202f-9ec4-4756-b15b-e9f152122250%22%2C+%22arrived_on%22%3A+%222017-10-04T07%3A15%3A17.548657Z%22%2C+%22left_on%22%3A+%222017-10-04T07%3A15%3A17.604668Z%22%2C+%22text%22%3A+%22Fruit%3F%22%2C+%22type%22%3A+%22A%22%2C+%22value%22%3A+null%7D%2C+%7B%22node%22%3A+%228037c12f-a277-4255-b630-6a03b035767a%22%2C+%22arrived_on%22%3A+%222017-10-04T07%3A15%3A17.604668Z%22%2C+%22left_on%22%3A+%222017-10-04T07%3A18%3A08.171069Z%22%2C+%22text%22%3A+%22orange%22%2C+%22type%22%3A+%22R%22%2C+%22value%22%3A+%22orange%22%7D%2C+%7B%22node%22%3A+%223b15df81-a0bd-4de7-8186-145ea3bb6c43%22%2C+%22arrived_on%22%3A+%222017-10-04T07%3A18%3A08.171069Z%22%2C+%22left_on%22%3A+null%2C+%22text%22%3A+null%2C+%22type%22%3A+%22A%22%2C+%22value%22%3A+null%7D%5D&flow_base_language=base&channel=-1" # noqa request = self.factory.post( - '/submission', data, - content_type='application/x-www-form-urlencoded') + "/submission", data, content_type="application/x-www-form-urlencoded" + ) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) - response = self.view(request, username=self.user.username, - xform_pk=self.xform.pk) - self.assertContains(response, 'Successful submission', status_code=201) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Location'], 'http://testserver/submission') + response = self.view( + request, username=self.user.username, xform_pk=self.xform.pk + ) + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Location"], "http://testserver/submission") def test_post_empty_submission(self): """ Test empty submission fails. """ - request = self.factory.post( - '/%s/submission' % self.user.username, {}) + request = self.factory.post("/%s/submission" % self.user.username, {}) request.user = AnonymousUser() response = self.view(request, username=self.user.username) - self.assertContains(response, 'No XML submission file.', - status_code=400) + self.assertContains(response, "No XML submission file.", status_code=400) def test_auth_submission_head_request(self): """ Test HEAD submission request. """ - request = self.factory.head('/submission') + request = self.factory.head("/submission") response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request, username=self.user.username) self.assertEqual(response.status_code, 204, response.data) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Location'], 'http://testserver/submission') + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Location"], "http://testserver/submission") def test_head_submission_anonymous(self): """ Test HEAD submission request for anonymous user. """ - request = self.factory.head('/%s/submission' % self.user.username) + request = self.factory.head("/%s/submission" % self.user.username) request.user = AnonymousUser() response = self.view(request, username=self.user.username) self.assertEqual(response.status_code, 204, response.data) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Location'], - 'http://testserver/%s/submission' - % self.user.username) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual( + response["Location"], "http://testserver/%s/submission" % self.user.username + ) def test_floip_format_submission(self): """ @@ -639,17 +715,20 @@ def test_floip_format_submission(self): # pylint: disable=C0301 data = '[["2017-05-23T13:35:37.119-04:00", 20394823948, 923842093, "ae54d3", "female", {"option_order": ["male", "female"]}]]' # noqa request = self.factory.post( - '/submission', data, - content_type='application/vnd.org.flowinterop.results+json') + "/submission", + data, + content_type="application/vnd.org.flowinterop.results+json", + ) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) - response = self.view(request, username=self.user.username, - xform_pk=self.xform.pk) - self.assertContains(response, 'Successful submission', status_code=201) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Location'], 'http://testserver/submission') + response = self.view( + request, username=self.user.username, xform_pk=self.xform.pk + ) + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Location"], "http://testserver/submission") def test_floip_format_submission_missing_column(self): """ @@ -658,16 +737,22 @@ def test_floip_format_submission_missing_column(self): # pylint: disable=C0301 data = '[["2017-05-23T13:35:37.119-04:00", 923842093, "ae54d3", "female", {"option_order": ["male", "female"]}]]' # noqa request = self.factory.post( - '/submission', data, - content_type='application/vnd.org.flowinterop.results+json') + "/submission", + data, + content_type="application/vnd.org.flowinterop.results+json", + ) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) - response = self.view(request, username=self.user.username, - xform_pk=self.xform.pk) - self.assertContains(response, "Wrong number of values (5) in row 0, " - "expecting 6 values", status_code=400) + response = self.view( + request, username=self.user.username, xform_pk=self.xform.pk + ) + self.assertContains( + response, + "Wrong number of values (5) in row 0, " "expecting 6 values", + status_code=400, + ) def test_floip_format_submission_not_list(self): """ @@ -676,17 +761,21 @@ def test_floip_format_submission_not_list(self): # pylint: disable=C0301 data = '{"option_order": ["male", "female"]}' # noqa request = self.factory.post( - '/submission', data, - content_type='application/vnd.org.flowinterop.results+json') + "/submission", + data, + content_type="application/vnd.org.flowinterop.results+json", + ) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) - response = self.view(request, username=self.user.username, - xform_pk=self.xform.pk) + response = self.view( + request, username=self.user.username, xform_pk=self.xform.pk + ) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, {u'non_field_errors': [ - u'Invalid format. Expecting a list.']}) + self.assertEqual( + response.data, {"non_field_errors": ["Invalid format. Expecting a list."]} + ) def test_floip_format_submission_is_valid_json(self): """ @@ -695,14 +784,17 @@ def test_floip_format_submission_is_valid_json(self): # pylint: disable=C0301 data = '"2017-05-23T13:35:37.119-04:00", 923842093, "ae54d3", "female", {"option_order": ["male", "female"]}' # noqa request = self.factory.post( - '/submission', data, - content_type='application/vnd.org.flowinterop.results+json') + "/submission", + data, + content_type="application/vnd.org.flowinterop.results+json", + ) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) - response = self.view(request, username=self.user.username, - xform_pk=self.xform.pk) + response = self.view( + request, username=self.user.username, xform_pk=self.xform.pk + ) self.assertContains(response, "Extra data", status_code=400) def test_floip_format_multiple_rows_submission(self): @@ -712,17 +804,20 @@ def test_floip_format_multiple_rows_submission(self): # pylint: disable=C0301 data = '[["2017-05-23T13:35:37.119-04:00", 20394823948, 923842093, "ae54d3", "female", {"option_order": ["male", "female"]}], ["2017-05-23T13:35:47.822-04:00", 20394823950, 923842093, "ae54d7", "chocolate", null ]]' # noqa request = self.factory.post( - '/submission', data, - content_type='application/vnd.org.flowinterop.results+json') + "/submission", + data, + content_type="application/vnd.org.flowinterop.results+json", + ) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) - response = self.view(request, username=self.user.username, - xform_pk=self.xform.pk) - self.assertContains(response, 'Successful submission', status_code=201) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Location'], 'http://testserver/submission') + response = self.view( + request, username=self.user.username, xform_pk=self.xform.pk + ) + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Location"], "http://testserver/submission") def test_floip_format_multiple_rows_instance(self): """ @@ -731,20 +826,24 @@ def test_floip_format_multiple_rows_instance(self): # pylint: disable=C0301 data = '[["2017-05-23T13:35:37.119-04:00", 20394823948, 923842093, "ae54d3", "female", {"option_order": ["male", "female"]}], ["2017-05-23T13:35:47.822-04:00", 20394823950, 923842093, "ae54d7", "chocolate", null ]]' # noqa request = self.factory.post( - '/submission', data, - content_type='application/vnd.org.flowinterop.results+json') + "/submission", + data, + content_type="application/vnd.org.flowinterop.results+json", + ) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) - response = self.view(request, username=self.user.username, - xform_pk=self.xform.pk) + response = self.view( + request, username=self.user.username, xform_pk=self.xform.pk + ) instance_json = Instance.objects.last().json data_responses = [i[4] for i in json.loads(data)] - self.assertTrue(any(i in data_responses - for i in instance_json.values())) + self.assertTrue(any(i in data_responses for i in instance_json.values())) - @mock.patch('onadata.apps.api.viewsets.xform_submission_viewset.SubmissionSerializer') # noqa + @mock.patch( + "onadata.apps.api.viewsets.xform_submission_viewset.SubmissionSerializer" + ) # noqa def test_post_submission_unreadable_post_error(self, MockSerializer): """ Test UnreadablePostError exception during submission.. @@ -752,23 +851,35 @@ def test_post_submission_unreadable_post_error(self, MockSerializer): MockSerializer.side_effect = UnreadablePostError() s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - with open(submission_path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} - request = self.factory.post( - '/%s/submission' % self.user.username, data) + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + with open(submission_path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} + request = self.factory.post("/%s/submission" % self.user.username, data) request.user = AnonymousUser() response = self.view(request, username=self.user.username) - self.assertContains(response, 'Unable to read submitted file', - status_code=400) - self.assertTrue(response.has_header('X-OpenRosa-Version')) + self.assertContains( + response, "Unable to read submitted file", status_code=400 + ) + self.assertTrue(response.has_header("X-OpenRosa-Version")) def test_post_submission_using_pk_while_anonymous(self): """ @@ -778,35 +889,44 @@ def test_post_submission_using_pk_while_anonymous(self): """ s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - with open(submission_path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} - request = self.factory.post( - f'/enketo/{self.xform.pk}/submission', data) + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + with open(submission_path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} + request = self.factory.post(f"/enketo/{self.xform.pk}/submission", data) count = Instance.objects.filter(xform=self.xform).count() request.user = AnonymousUser() response = self.view(request, xform_pk=self.xform.pk) - self.assertContains(response, 'Successful submission', - status_code=201) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'text/xml; charset=utf-8') + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "text/xml; charset=utf-8") self.assertEqual( - response['Location'], - f'http://testserver/enketo/{self.xform.pk}/submission') + response["Location"], + f"http://testserver/enketo/{self.xform.pk}/submission", + ) self.assertEqual( - Instance.objects.filter(xform=self.xform).count(), - count+1) + Instance.objects.filter(xform=self.xform).count(), count + 1 + ) def test_head_submission_request_w_no_auth(self): """ @@ -819,37 +939,48 @@ def test_head_submission_request_w_no_auth(self): s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - with open(submission_path, 'rb'): + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + with open(submission_path, "rb"): # When require_auth is enabled and # no auth is passed to the request should fail - request = self.factory.head( - f'/enketo/{self.xform.pk}/submission') + request = self.factory.head(f"/enketo/{self.xform.pk}/submission") response = self.view(request) self.assertEqual(response.status_code, 401) # When require_auth is enabled & no auth passed is ok - request = self.factory.head( - f'/enketo/{self.xform.pk}/submission') + request = self.factory.head(f"/enketo/{self.xform.pk}/submission") request.user = self.xform.user response = self.view(request, xform_pk=self.xform.pk) self.assertEqual(response.status_code, 401) - self.assertTrue(response.has_header('X-OpenRosa-Version')) + self.assertTrue(response.has_header("X-OpenRosa-Version")) # Test Content-Length header is available - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Location'], - f'http://testserver/enketo/{self.xform.pk}/submission') # noqa + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual( + response["Location"], + f"http://testserver/enketo/{self.xform.pk}/submission", + ) # noqa def test_post_submission_using_pk_while_authenticated(self): """ @@ -859,35 +990,44 @@ def test_post_submission_using_pk_while_authenticated(self): """ s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - with open(submission_path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + with open(submission_path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} count = Instance.objects.filter(xform=self.xform).count() - request = self.factory.post( - f'/enketo/{self.xform.pk}/submission', data) + request = self.factory.post(f"/enketo/{self.xform.pk}/submission", data) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request, form_pk=self.xform.pk) - self.assertContains(response, 'Successful submission', - status_code=201) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'text/xml; charset=utf-8') + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "text/xml; charset=utf-8") self.assertEqual( - response['Location'], - f'http://testserver/enketo/{self.xform.pk}/submission') + response["Location"], + f"http://testserver/enketo/{self.xform.pk}/submission", + ) self.assertEqual( - Instance.objects.filter(xform=self.xform).count(), - count + 1) + Instance.objects.filter(xform=self.xform).count(), count + 1 + ) diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index 37b8622934..032505f875 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -49,7 +49,10 @@ xls_url_no_extension_mock_content_disposition_attr_jumbled_v1, xls_url_no_extension_mock_content_disposition_attr_jumbled_v2, ) -from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import ( + TestAbstractViewSet, + get_mocked_response_for_file, +) from onadata.apps.api.viewsets.project_viewset import ProjectViewSet from onadata.apps.api.viewsets.xform_viewset import XFormViewSet from onadata.apps.logger.models import Attachment @@ -90,7 +93,14 @@ ) +ROLES = [ReadOnlyRole, DataEntryRole, EditorRole, ManagerRole, OwnerRole] + +JWT_SECRET_KEY = "thesecretkey" +JWT_ALGORITHM = "HS256" + + def fixtures_path(filepath): + """Returns the file object at the given filepath.""" return open( os.path.join( settings.PROJECT_ROOT, "libs", "tests", "utils", "fixtures", filepath @@ -106,13 +116,9 @@ def raise_bad_status_line(arg): raise BadStatusLine("RANDOM STATUS") -ROLES = [ReadOnlyRole, DataEntryRole, EditorRole, ManagerRole, OwnerRole] - -JWT_SECRET_KEY = "thesecretkey" -JWT_ALGORITHM = "HS256" - - class TestXFormViewSet(TestAbstractViewSet): + """Test XFormViewSet""" + def setUp(self): super(TestXFormViewSet, self).setUp() self.view = XFormViewSet.as_view( @@ -1027,9 +1033,9 @@ def test_enketo_urls_remain_the_same_after_form_replacement(self): self.xform.refresh_from_db() # diff versions - self.assertNotEquals(version, self.xform.version) - self.assertEquals(form_id, self.xform.pk) - self.assertEquals(id_string, self.xform.id_string) + self.assertNotEqual(version, self.xform.version) + self.assertEqual(form_id, self.xform.pk) + self.assertEqual(id_string, self.xform.id_string) def test_xform_hash_changes_after_form_replacement(self): with HTTMock(enketo_mock): @@ -1061,7 +1067,7 @@ def test_xform_hash_changes_after_form_replacement(self): self.assertEqual(response.status_code, 200) self.xform.refresh_from_db() - self.assertNotEquals(xform_old_hash, self.xform.hash) + self.assertNotEqual(xform_old_hash, self.xform.hash) def test_hash_changes_after_update_xform_xls_file(self): with HTTMock(enketo_mock): @@ -1092,7 +1098,7 @@ def test_hash_changes_after_update_xform_xls_file(self): self.assertEqual(response.status_code, 200) self.xform.refresh_from_db() - self.assertNotEquals(xform_old_hash, self.xform.hash) + self.assertNotEqual(xform_old_hash, self.xform.hash) def test_login_enketo_no_redirect(self): with HTTMock(enketo_urls_mock): @@ -1242,7 +1248,7 @@ def test_publish_xlsform(self): self.assertDictContainsSubset(data, response.data) self.assertTrue(OwnerRole.user_has_role(self.user, xform)) - self.assertEquals("owner", response.data["users"][0]["role"]) + self.assertEqual("owner", response.data["users"][0]["role"]) # pylint: disable=no-member self.assertIsNotNone( @@ -1298,7 +1304,7 @@ def test_publish_xlsforms_with_same_id_string(self): ) self.assertDictContainsSubset(data, response.data) self.assertTrue(OwnerRole.user_has_role(self.user, xform)) - self.assertEquals("owner", response.data["users"][0]["role"]) + self.assertEqual("owner", response.data["users"][0]["role"]) # pylint: disable=no-member self.assertIsNotNone( @@ -1339,7 +1345,7 @@ def test_publish_xlsforms_with_same_id_string(self): self.assertDictContainsSubset(data, response.data) self.assertTrue(OwnerRole.user_has_role(self.user, xform)) - self.assertEquals("owner", response.data["users"][0]["role"]) + self.assertEqual("owner", response.data["users"][0]["role"]) # pylint: disable=no-member self.assertIsNotNone( @@ -1355,8 +1361,9 @@ def test_publish_xlsforms_with_same_id_string(self): self.assertIsInstance(xform, XForm) self.assertEqual(counter + 2, XForm.objects.count()) - @patch("onadata.apps.main.forms.urlopen") - def test_publish_xlsform_using_url_upload(self, mock_urlopen): + # pylint: disable=invalid-name + @patch("onadata.apps.main.forms.requests") + def test_publish_xlsform_using_url_upload(self, mock_requests): with HTTMock(enketo_mock): view = XFormViewSet.as_view({"post": "create"}) @@ -1372,21 +1379,26 @@ def test_publish_xlsform_using_url_upload(self, mock_urlopen): "transportation_different_id_string.xlsx", ) - xls_file = open(path, "rb") - mock_urlopen.return_value = xls_file + with open(path, "rb") as xls_file: + mock_response = get_mocked_response_for_file( + xls_file, "transportation_different_id_string.xlsx", 200 + ) + mock_requests.head.return_value = mock_response + mock_requests.get.return_value = mock_response - post_data = {"xls_url": xls_url} - request = self.factory.post("/", data=post_data, **self.extra) - response = view(request) + post_data = {"xls_url": xls_url} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request) - mock_urlopen.assert_called_with(xls_url) - xls_file.close() + mock_requests.get.assert_called_with(xls_url) + xls_file.close() - self.assertEqual(response.status_code, 201) - self.assertEqual(XForm.objects.count(), pre_count + 1) + self.assertEqual(response.status_code, 201) + self.assertEqual(XForm.objects.count(), pre_count + 1) - @patch("onadata.apps.main.forms.urlopen") - def test_publish_xlsform_using_url_with_no_extension(self, mock_urlopen): + # pylint: disable=invalid-name + @patch("onadata.apps.main.forms.requests") + def test_publish_xlsform_using_url_with_no_extension(self, mock_requests): with HTTMock(enketo_mock, xls_url_no_extension_mock): view = XFormViewSet.as_view({"post": "create"}) @@ -1402,19 +1414,24 @@ def test_publish_xlsform_using_url_with_no_extension(self, mock_urlopen): "transportation_different_id_string.xlsx", ) - xls_file = open(path, "rb") - mock_urlopen.return_value = xls_file + with open(path, "rb") as xls_file: + mock_response = get_mocked_response_for_file( + xls_file, "transportation_version.xlsx", 200 + ) + mock_requests.head.return_value = mock_response + mock_requests.get.return_value = mock_response - post_data = {"xls_url": xls_url} - request = self.factory.post("/", data=post_data, **self.extra) - response = view(request) + post_data = {"xls_url": xls_url} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request) - self.assertEqual(response.status_code, 201) - self.assertEqual(XForm.objects.count(), pre_count + 1) + self.assertEqual(response.status_code, 201, response.data) + self.assertEqual(XForm.objects.count(), pre_count + 1) - @patch("onadata.apps.main.forms.urlopen") + # pylint: disable=invalid-name + @patch("onadata.apps.main.forms.requests") def test_publish_xlsform_using_url_content_disposition_attr_jumbled_v1( - self, mock_urlopen + self, mock_requests ): with HTTMock( enketo_mock, xls_url_no_extension_mock_content_disposition_attr_jumbled_v1 @@ -1433,19 +1450,24 @@ def test_publish_xlsform_using_url_content_disposition_attr_jumbled_v1( "transportation_different_id_string.xlsx", ) - xls_file = open(path, "rb") - mock_urlopen.return_value = xls_file + with open(path, "rb") as xls_file: + mock_response = get_mocked_response_for_file( + xls_file, "transportation_different_id_string.xlsx", 200 + ) + mock_requests.head.return_value = mock_response + mock_requests.get.return_value = mock_response - post_data = {"xls_url": xls_url} - request = self.factory.post("/", data=post_data, **self.extra) - response = view(request) + post_data = {"xls_url": xls_url} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request) - self.assertEqual(response.status_code, 201) - self.assertEqual(XForm.objects.count(), pre_count + 1) + self.assertEqual(response.status_code, 201) + self.assertEqual(XForm.objects.count(), pre_count + 1) - @patch("onadata.apps.main.forms.urlopen") + # pylint: disable=invalid-name + @patch("onadata.apps.main.forms.requests") def test_publish_xlsform_using_url_content_disposition_attr_jumbled_v2( - self, mock_urlopen + self, mock_requests ): with HTTMock( enketo_mock, xls_url_no_extension_mock_content_disposition_attr_jumbled_v2 @@ -1464,18 +1486,23 @@ def test_publish_xlsform_using_url_content_disposition_attr_jumbled_v2( "transportation_different_id_string.xlsx", ) - xls_file = open(path, "rb") - mock_urlopen.return_value = xls_file + with open(path, "rb") as xls_file: + mock_response = get_mocked_response_for_file( + xls_file, "transportation_different_id_string.xlsx", 200 + ) + mock_requests.head.return_value = mock_response + mock_requests.get.return_value = mock_response - post_data = {"xls_url": xls_url} - request = self.factory.post("/", data=post_data, **self.extra) - response = view(request) + post_data = {"xls_url": xls_url} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request) - self.assertEqual(response.status_code, 201) - self.assertEqual(XForm.objects.count(), pre_count + 1) + self.assertEqual(response.status_code, 201) + self.assertEqual(XForm.objects.count(), pre_count + 1) - @patch("onadata.apps.main.forms.urlopen") - def test_publish_csvform_using_url_upload(self, mock_urlopen): + # pylint: disable=invalid-name + @patch("onadata.apps.main.forms.requests") + def test_publish_csvform_using_url_upload(self, mock_requests): with HTTMock(enketo_mock): view = XFormViewSet.as_view({"post": "create"}) @@ -1490,19 +1517,24 @@ def test_publish_csvform_using_url_upload(self, mock_urlopen): "text_and_integer.csv", ) - csv_file = open(path, "rb") - mock_urlopen.return_value = csv_file + with open(path, "rb") as csv_file: + mock_response = get_mocked_response_for_file( + csv_file, "text_and_integer.csv", 200 + ) + mock_requests.head.return_value = mock_response + mock_requests.get.return_value = mock_response - post_data = {"csv_url": csv_url} - request = self.factory.post("/", data=post_data, **self.extra) - response = view(request) + post_data = {"csv_url": csv_url} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request) - mock_urlopen.assert_called_with(csv_url) - csv_file.close() + mock_requests.get.assert_called_with(csv_url) + csv_file.close() - self.assertEqual(response.status_code, 201) - self.assertEqual(XForm.objects.count(), pre_count + 1) + self.assertEqual(response.status_code, 201) + self.assertEqual(XForm.objects.count(), pre_count + 1) + # pylint: disable=invalid-name def test_publish_select_external_xlsform(self): with HTTMock(enketo_urls_mock): view = XFormViewSet.as_view({"post": "create"}) @@ -1528,7 +1560,7 @@ def test_publish_select_external_xlsform(self): ) self.assertIsNotNone(metadata) self.assertTrue(OwnerRole.user_has_role(self.user, xform)) - self.assertEquals("owner", response.data["users"][0]["role"], self.user) + self.assertEqual("owner", response.data["users"][0]["role"], self.user) def test_publish_csv_with_universal_newline_xlsform(self): with HTTMock(enketo_mock): @@ -1796,18 +1828,17 @@ def test_form_add_project_cache(self): with HTTMock(enketo_mock): self._project_create() + cleared_cache_content = ["forms"] # set project XForm cache - cache.set("{}{}".format(PROJ_FORMS_CACHE, self.project.pk), ["forms"]) + cache.set(f"{PROJ_FORMS_CACHE}{self.project.pk}", cleared_cache_content) - self.assertNotEqual( - cache.get("{}{}".format(PROJ_FORMS_CACHE, self.project.pk)), None - ) + self.assertNotEqual(cache.get(f"{PROJ_FORMS_CACHE}{self.project.pk}"), None) self._publish_xls_form_to_project() - # test project XForm cache is empty - self.assertEqual( - cache.get("{}{}".format(PROJ_FORMS_CACHE, self.project.pk)), None + # test project XForm cache has new content + self.assertNotEqual( + cache.get(f"{PROJ_FORMS_CACHE}{self.project.pk}"), cleared_cache_content ) def test_form_delete(self): @@ -1822,11 +1853,9 @@ def test_form_delete(self): self.assertNotEqual(etag_value, None) # set project XForm cache - cache.set("{}{}".format(PROJ_FORMS_CACHE, self.project.pk), ["forms"]) + cache.set(f"{PROJ_FORMS_CACHE}{self.project.pk}", ["forms"]) - self.assertNotEqual( - cache.get("{}{}".format(PROJ_FORMS_CACHE, self.project.pk)), None - ) + self.assertNotEqual(cache.get(f"{PROJ_FORMS_CACHE}{self.project.pk}"), None) view = XFormViewSet.as_view({"delete": "destroy", "get": "retrieve"}) formid = self.xform.pk @@ -1836,9 +1865,7 @@ def test_form_delete(self): self.assertEqual(response.status_code, 204) # test project XForm cache is emptied - self.assertEqual( - cache.get("{}{}".format(PROJ_FORMS_CACHE, self.project.pk)), None - ) + self.assertEqual(cache.get(f"{PROJ_FORMS_CACHE}{self.project.pk}"), None) self.xform.refresh_from_db() @@ -1974,8 +2001,8 @@ def test_form_clone_endpoint(self): response = view(request, pk=formid) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data["project"], - ["invalid literal for int() with base 10: 'abc123'"], + str(response.data["project"]), + "[ErrorDetail(string=\"Field 'id' expected a number but got 'abc123'.\", code='invalid')]", ) # pylint: disable=no-member @@ -2104,7 +2131,7 @@ def test_external_export(self): response = view(request, pk=formid, format="xls") self.assertEqual(response.status_code, 302) expected_url = "http://xls_server/xls/ee3ff9d8f5184fc4a8fdebc2547cc059" - self.assertEquals(response.url, expected_url) + self.assertEqual(response.url, expected_url) def test_external_export_with_data_id(self): with HTTMock(enketo_mock): @@ -2146,7 +2173,7 @@ def test_external_export_with_data_id(self): response = view(request, pk=formid, format="xls") self.assertEqual(response.status_code, 302) expected_url = "http://xls_server/xls/ee3ff9d8f5184fc4a8fdebc2547cc059" - self.assertEquals(response.url, expected_url) + self.assertEqual(response.url, expected_url) def test_external_export_error(self): with HTTMock(enketo_mock): @@ -2322,7 +2349,7 @@ def test_csv_import_diff_column(self): response = view(request, pk=self.xform.id) self.assertEqual(response.status_code, 400) self.assertIn("error", response.data) - self.assertEquals( + self.assertEqual( response.data.get("error"), "Sorry uploaded file does not match the form. " "The file is missing the column(s): age, name.", @@ -2347,7 +2374,7 @@ def test_csv_import_additional_columns(self): self.assertEqual(response.status_code, 200) self.assertIn("info", response.data) - self.assertEquals( + self.assertEqual( response.data.get("info"), "Additional column(s) excluded from the upload:" " '_additional'.", ) @@ -2480,11 +2507,11 @@ def test_update_xform_xls_file(self): new_version = self.xform.version new_last_updated_at = self.xform.last_updated_at # diff versions - self.assertNotEquals(last_updated_at, new_last_updated_at) - self.assertNotEquals(version, new_version) - self.assertNotEquals(title_old, self.xform.title) - self.assertEquals(form_id, self.xform.pk) - self.assertEquals(id_string, self.xform.id_string) + self.assertNotEqual(last_updated_at, new_last_updated_at) + self.assertNotEqual(version, new_version) + self.assertNotEqual(title_old, self.xform.title) + self.assertEqual(form_id, self.xform.pk) + self.assertEqual(id_string, self.xform.id_string) def test_manager_can_update_xform_xls_file(self): """Manager Role can replace xlsform""" @@ -2536,10 +2563,10 @@ def test_manager_can_update_xform_xls_file(self): new_version = self.xform.version # diff versions - self.assertNotEquals(version, new_version) - self.assertNotEquals(title_old, self.xform.title) - self.assertEquals(form_id, self.xform.pk) - self.assertEquals(id_string, self.xform.id_string) + self.assertNotEqual(version, new_version) + self.assertNotEqual(title_old, self.xform.title) + self.assertEqual(form_id, self.xform.pk) + self.assertEqual(id_string, self.xform.id_string) xml = self.xform.xml fhuuid = xml.find("formhub/uuid") self.assertEqual(xml[xml[:fhuuid].rfind("=") + 2 : fhuuid], "/data/") @@ -2802,8 +2829,8 @@ def test_update_xform_xls_bad_file(self): new_version = self.xform.version # fails to update the form - self.assertEquals(version, new_version) - self.assertEquals(form_id, self.xform.pk) + self.assertEqual(version, new_version) + self.assertEqual(form_id, self.xform.pk) def test_update_xform_xls_file_with_submissions(self): with HTTMock(enketo_mock): @@ -2841,10 +2868,10 @@ def test_update_xform_xls_file_with_submissions(self): self.xform.refresh_from_db() - self.assertEquals(form_id, self.xform.pk) - self.assertNotEquals(version, self.xform.version) - self.assertNotEquals(xform_json, self.xform.json) - self.assertNotEquals(xform_xml, self.xform.xml) + self.assertEqual(form_id, self.xform.pk) + self.assertNotEqual(version, self.xform.version) + self.assertNotEqual(xform_json, self.xform.json) + self.assertNotEqual(xform_xml, self.xform.xml) is_updated_form = ( len( [ @@ -2889,11 +2916,11 @@ def test_update_xform_xls_file_with_version_set(self): self.xform.refresh_from_db() # diff versions - self.assertEquals(self.xform.version, "212121211") - self.assertEquals(form_id, self.xform.pk) + self.assertEqual(self.xform.version, "212121211") + self.assertEqual(form_id, self.xform.pk) - @patch("onadata.apps.main.forms.urlopen") - def test_update_xform_xls_url(self, mock_urlopen): + @patch("onadata.apps.main.forms.requests") + def test_update_xform_xls_url(self, mock_requests): with HTTMock(enketo_mock): self._publish_xls_form_to_project() form_id = self.xform.pk @@ -2918,24 +2945,28 @@ def test_update_xform_xls_url(self, mock_urlopen): "transportation_version.xlsx", ) - xls_file = open(path, "rb") - mock_urlopen.return_value = xls_file + with open(path, "rb") as xls_file: + mock_response = get_mocked_response_for_file( + xls_file, "transportation_version.xlsx", 200 + ) + mock_requests.head.return_value = mock_response + mock_requests.get.return_value = mock_response - post_data = {"xls_url": xls_url} - request = self.factory.patch("/", data=post_data, **self.extra) - response = view(request, pk=form_id) + post_data = {"xls_url": xls_url} + request = self.factory.patch("/", data=post_data, **self.extra) + response = view(request, pk=form_id) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 200) - self.xform.refresh_from_db() + self.xform.refresh_from_db() - self.assertEquals(count, XForm.objects.all().count()) - # diff versions - self.assertEquals(self.xform.version, "212121211") - self.assertEquals(form_id, self.xform.pk) + self.assertEqual(count, XForm.objects.all().count()) + # diff versions + self.assertEqual(self.xform.version, "212121211") + self.assertEqual(form_id, self.xform.pk) - @patch("onadata.apps.main.forms.urlopen") - def test_update_xform_dropbox_url(self, mock_urlopen): + @patch("onadata.apps.main.forms.requests") + def test_update_xform_dropbox_url(self, mock_requests): with HTTMock(enketo_mock): self._publish_xls_form_to_project() form_id = self.xform.pk @@ -2960,21 +2991,25 @@ def test_update_xform_dropbox_url(self, mock_urlopen): "transportation_version.xlsx", ) - xls_file = open(path, "rb") - mock_urlopen.return_value = xls_file + with open(path, "rb") as xls_file: + mock_response = get_mocked_response_for_file( + xls_file, "transportation_version.xlsx", 200 + ) + mock_requests.head.return_value = mock_response + mock_requests.get.return_value = mock_response - post_data = {"dropbox_xls_url": xls_url} - request = self.factory.patch("/", data=post_data, **self.extra) - response = view(request, pk=form_id) + post_data = {"dropbox_xls_url": xls_url} + request = self.factory.patch("/", data=post_data, **self.extra) + response = view(request, pk=form_id) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 200) - self.xform.refresh_from_db() + self.xform.refresh_from_db() - self.assertEquals(count, XForm.objects.all().count()) - # diff versions - self.assertEquals(self.xform.version, "212121211") - self.assertEquals(form_id, self.xform.pk) + self.assertEqual(count, XForm.objects.all().count()) + # diff versions + self.assertEqual(self.xform.version, "212121211") + self.assertEqual(form_id, self.xform.pk) def test_update_xform_using_put_with_invalid_input(self): with HTTMock(enketo_mock): @@ -3078,7 +3113,7 @@ def test_update_xform_using_put_with_invalid_input(self): self.xform.refresh_from_db() # check that description has been sanitized - self.assertEquals( + self.assertEqual( conditional_escape(unsanitized_html_str), self.xform.description ) @@ -3114,10 +3149,10 @@ def test_update_xform_using_put(self): self.xform.refresh_from_db() - self.assertEquals(version, self.xform.version) - self.assertEquals(self.xform.description, "Transport form") - self.assertEquals(self.xform.title, "Transport Form") - self.assertEquals(form_id, self.xform.pk) + self.assertEqual(version, self.xform.version) + self.assertEqual(self.xform.description, "Transport form") + self.assertEqual(self.xform.title, "Transport Form") + self.assertEqual(form_id, self.xform.pk) def test_update_xform_using_put_without_required_field(self): with HTTMock(enketo_mock): @@ -3147,7 +3182,7 @@ def test_update_xform_using_put_without_required_field(self): self.assertEqual(response.status_code, 400) self.assertEqual(response.get("Cache-Control"), None) - self.assertEquals(response.data, {"title": ["This field is required."]}) + self.assertEqual(response.data, {"title": ["This field is required."]}) def test_public_xform_accessible_by_authenticated_users(self): with HTTMock(enketo_mock): @@ -3189,7 +3224,7 @@ def test_publish_form_async(self, mock_get_status): self.assertTrue("job_uuid" in response.data) - self.assertEquals(count + 1, XForm.objects.count()) + self.assertEqual(count + 1, XForm.objects.count()) # get the result get_data = {"job_uuid": response.data.get("job_uuid")} @@ -3199,7 +3234,7 @@ def test_publish_form_async(self, mock_get_status): self.assertTrue(mock_get_status.called) self.assertEqual(response.status_code, 202) - self.assertEquals(response.data, {"job_status": "PENDING"}) + self.assertEqual(response.data, {"job_status": "PENDING"}) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) @patch( @@ -3289,7 +3324,7 @@ def test_failed_form_publishing_after_maximum_retries( response = view(request) self.assertEqual(response.status_code, 202) - self.assertEquals(response.data, error_message) + self.assertEqual(response.data, error_message) def test_survey_preview_endpoint(self): view = XFormViewSet.as_view({"post": "survey_preview", "get": "survey_preview"}) @@ -3371,7 +3406,7 @@ def test_delete_xform_async(self, mock_get_status): self.assertEqual(response.status_code, 202) self.assertTrue("job_uuid" in response.data) self.assertTrue("time_async_triggered" in response.data) - self.assertEquals(count, XForm.objects.count()) + self.assertEqual(count, XForm.objects.count()) view = XFormViewSet.as_view({"get": "delete_async"}) @@ -3381,7 +3416,7 @@ def test_delete_xform_async(self, mock_get_status): self.assertTrue(mock_get_status.called) self.assertEqual(response.status_code, 202) - self.assertEquals(response.data, {"job_status": "PENDING"}) + self.assertEqual(response.data, {"job_status": "PENDING"}) xform = XForm.objects.get(pk=formid) @@ -3486,8 +3521,6 @@ def test_export_zip_async(self, async_result): @patch("onadata.libs.utils.api_export_tools.AsyncResult") def test_export_async_connection_error(self, async_result): with HTTMock(enketo_mock): - from requests import ConnectionError - async_result.side_effect = ConnectionError( "Error opening socket: a socket error occurred" ) @@ -3641,7 +3674,7 @@ def test_check_async_publish_empty_uuid(self): response = view(request) self.assertEqual(response.status_code, 202) - self.assertEquals(response.data, {"error": "Empty job uuid"}) + self.assertEqual(response.data, {"error": "Empty job uuid"}) def test_always_new_report_with_data_id(self): with HTTMock(enketo_mock): @@ -3684,7 +3717,7 @@ def test_always_new_report_with_data_id(self): response = view(request, pk=formid, format="xls") self.assertEqual(response.status_code, 302) expected_url = "http://xls_server/xls/ee3ff9d8f5184fc4a8fdebc2547cc059" - self.assertEquals(response.url, expected_url) + self.assertEqual(response.url, expected_url) count = Export.objects.filter( xform=self.xform, export_type=Export.EXTERNAL_EXPORT @@ -3695,13 +3728,13 @@ def test_always_new_report_with_data_id(self): response = view(request, pk=formid, format="xls") self.assertEqual(response.status_code, 302) expected_url = "http://xls_server/xls/ee3ff9d8f5184fc4a8fdebc2547cc057" - self.assertEquals(response.url, expected_url) + self.assertEqual(response.url, expected_url) count2 = Export.objects.filter( xform=self.xform, export_type=Export.EXTERNAL_EXPORT ).count() - self.assertEquals(count + 1, count2) + self.assertEqual(count + 1, count2) def test_different_form_versions(self): with HTTMock(enketo_mock): @@ -3820,7 +3853,7 @@ def test_versions_endpoint(self): returned_data.pop("date_modified") self.assertEqual(returned_data, expected_data) old_version = self.xform.version - expected_json = self.xform.json + expected_json = self.xform.json_dict() expected_xml = self.xform.xml # Replace form @@ -3846,7 +3879,7 @@ def test_versions_endpoint(self): ) response = view(request, pk=self.xform.pk, version_id=old_version) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data, json.loads(expected_json)) + self.assertEqual(response.data, expected_json) response = view(request, pk=self.xform.pk, version_id=old_version, format="xml") self.assertEqual(response.status_code, 200) @@ -4254,7 +4287,7 @@ def test_csv_export_no_new_generated(self): response = view(request, pk=self.xform.pk, format="csv") self.assertEqual(response.status_code, 200) - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) headers = dict(response.items()) self.assertEqual(headers["Content-Type"], "application/csv") @@ -4269,7 +4302,7 @@ def test_csv_export_no_new_generated(self): self.assertEqual(response.status_code, 200) # no new export generated - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) headers = dict(response.items()) self.assertEqual(headers["Content-Type"], "application/csv") @@ -4346,8 +4379,8 @@ def test_xform_linked_dataviews(self): data = { "name": "Another DataView", - "xform": "http://testserver/api/v1/forms/%s" % self.xform.pk, - "project": "http://testserver/api/v1/projects/%s" % self.project.pk, + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", "columns": '["name", "age", "gender"]', "query": '[{"column":"age","filter":">","value":"50"}]', } @@ -4366,7 +4399,7 @@ def test_xform_linked_dataviews(self): self.assertEqual(response.status_code, 200) self.assertIn("data_views", response.data) - self.assertEquals(2, len(response.data["data_views"])) + self.assertEqual(2, len(response.data["data_views"])) def test_delete_xform_also_deletes_linked_dataviews(self): """ @@ -4397,8 +4430,8 @@ def test_delete_xform_also_deletes_linked_dataviews(self): self._create_dataview() data = { "name": "Another DataView", - "xform": "http://testserver/api/v1/forms/%s" % self.xform.pk, - "project": "http://testserver/api/v1/projects/%s" % self.project.pk, + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", "columns": '["name", "age", "gender"]', "query": '[{"column":"age","filter":">","value":"50"}]', } @@ -4415,7 +4448,7 @@ def test_delete_xform_also_deletes_linked_dataviews(self): response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertIn("data_views", response.data) - self.assertEquals(2, len(response.data["data_views"])) + self.assertEqual(2, len(response.data["data_views"])) # delete xform view = XFormViewSet.as_view({"delete": "destroy", "get": "retrieve"}) @@ -4450,10 +4483,10 @@ def test_multitple_enketo_urls(self): count = MetaData.objects.filter( object_id=self.xform.id, data_type="enketo_url" ).count() - self.assertEquals(2, count) + self.assertEqual(2, count) # delete cache - safe_delete("{}{}".format(ENKETO_URL_CACHE, self.xform.pk)) + safe_delete(f"{ENKETO_URL_CACHE}{self.xform.pk}") view = XFormViewSet.as_view( { @@ -4536,7 +4569,7 @@ def test_csv_export_filtered_by_date(self): count = Export.objects.all().count() response = self._get_date_filtered_export(query_str) - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) test_file_path = os.path.join( settings.PROJECT_ROOT, @@ -4554,7 +4587,7 @@ def test_csv_export_filtered_by_date(self): export = Export.objects.last() self.assertIn("query", export.options) - self.assertEquals(export.options["query"], query_str) + self.assertEqual(export.options["query"], query_str) @patch("onadata.libs.utils.api_export_tools.AsyncResult") def test_export_form_data_async_with_filtered_date(self, async_result): @@ -4590,7 +4623,7 @@ def test_export_form_data_async_with_filtered_date(self, async_result): self.assertIsNotNone(response.data) self.assertEqual(response.status_code, 202) self.assertTrue("job_uuid" in response.data) - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) task_id = response.data.get("job_uuid") get_data = {"job_uuid": task_id} @@ -4604,7 +4637,7 @@ def test_export_form_data_async_with_filtered_date(self, async_result): export = Export.objects.last() self.assertIn("query", export.options) - self.assertEquals(export.options["query"], query_str) + self.assertEqual(export.options["query"], query_str) def test_previous_export_with_date_filter_is_returned(self): with HTTMock(enketo_mock): @@ -4631,7 +4664,7 @@ def test_previous_export_with_date_filter_is_returned(self): self._get_date_filtered_export(query_str) # no change in count of exports - self.assertEquals(count, Export.objects.all().count()) + self.assertEqual(count, Export.objects.all().count()) def test_download_export_with_export_id(self): with HTTMock(enketo_mock): @@ -4715,7 +4748,7 @@ def test_normal_export_after_export_with_date_filter(self): self.assertEqual(response.status_code, 200) # should create a new export - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) @patch("onadata.libs.utils.api_export_tools.AsyncResult") def test_export_form_data_async_include_labels(self, async_result): @@ -5081,7 +5114,7 @@ def test_xform_version_count(self): instance.set_deleted() # delete cache - safe_delete("{}{}".format(XFORM_DATA_VERSIONS, self.xform.pk)) + safe_delete(f"{XFORM_DATA_VERSIONS}{self.xform.pk}") request = self.factory.get("/", **self.extra) response = view(request, pk=self.xform.pk) @@ -5208,7 +5241,7 @@ def test_csv_export_cache(self): self.assertEqual(response.status_code, 200) # should generate new - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) survey = self.surveys[0] self._make_submission( @@ -5232,7 +5265,7 @@ def test_csv_export_cache(self): self.assertEqual(response.status_code, 200) # changed options, should generate new - self.assertEquals(count + 2, Export.objects.all().count()) + self.assertEqual(count + 2, Export.objects.all().count()) data = {"export_type": "csv", "win_excel_utf8": False} @@ -5241,7 +5274,7 @@ def test_csv_export_cache(self): self.assertEqual(response.status_code, 200) # reused options, should generate new with new submission - self.assertEquals(count + 3, Export.objects.all().count()) + self.assertEqual(count + 3, Export.objects.all().count()) def test_upload_xml_form_file(self): with HTTMock(enketo_mock): @@ -5428,24 +5461,6 @@ def test_csv_xls_import_errors(self): self.assertEqual(response.status_code, 400) self.assertEqual(response.data.get("error"), "csv_file not a csv file") - @patch("onadata.apps.main.forms.urlopen", side_effect=raise_bad_status_line) - def test_error_raised_xform_url_upload_urllib_error(self, mock_urlopen): - """ - Test that the BadStatusLine error thrown by urlopen when a status - code is not understood is handled properly - """ - view = XFormViewSet.as_view({"post": "create"}) - - xls_url = "http://localhost:2000" - - post_data = {"xls_url": xls_url} - request = self.factory.post("/", data=post_data, **self.extra) - response = view(request) - error_msg = "An error occurred while publishing the form. " "Please try again." - - self.assertEqual(response.status_code, 400) - self.assertEqual(response.data.get("text"), error_msg) - def test_export_csvzip_form_data_async(self): with HTTMock(enketo_mock): xls_path = os.path.join( diff --git a/onadata/apps/api/tools.py b/onadata/apps/api/tools.py index d6ed815c01..798e5d21e0 100644 --- a/onadata/apps/api/tools.py +++ b/onadata/apps/api/tools.py @@ -8,7 +8,8 @@ from django import forms from django.conf import settings -from django.contrib.auth.models import Permission, User +from django.contrib.auth import get_user_model +from django.contrib.auth.models import Permission from django.contrib.contenttypes.models import ContentType from django.contrib.sites.models import Site from django.core.files.storage import get_storage_class @@ -19,19 +20,21 @@ from django.db.utils import IntegrityError from django.http import HttpResponseNotFound, HttpResponseRedirect from django.shortcuts import get_object_or_404 -from django.utils.translation import ugettext as _ from django.utils.module_loading import import_string -from future.utils import listitems -from guardian.shortcuts import assign_perm, get_perms_for_model, remove_perm -from guardian.shortcuts import get_perms +from django.utils.translation import gettext as _ + +from guardian.shortcuts import assign_perm, get_perms, get_perms_for_model, remove_perm from kombu.exceptions import OperationalError +from multidb.pinning import use_master from registration.models import RegistrationProfile from rest_framework import exceptions +from six import iteritems from taggit.forms import TagField -from multidb.pinning import use_master from onadata.apps.api.models.organization_profile import ( - OrganizationProfile, create_owner_team_and_assign_permissions) + OrganizationProfile, + create_owner_team_and_assign_permissions, +) from onadata.apps.api.models.team import Team from onadata.apps.logger.models import DataView, Instance, Project, XForm from onadata.apps.main.forms import QuickConverter @@ -41,39 +44,66 @@ from onadata.libs.baseviewset import DefaultBaseViewset from onadata.libs.models.share_project import ShareProject from onadata.libs.permissions import ( - ROLES, DataEntryMinorRole, DataEntryOnlyRole, DataEntryRole, - EditorMinorRole, EditorRole, ManagerRole, OwnerRole, get_role, - get_role_in_org, is_organization) + ROLES, + DataEntryMinorRole, + DataEntryOnlyRole, + DataEntryRole, + EditorMinorRole, + EditorRole, + ManagerRole, + OwnerRole, + get_role, + get_role_in_org, + is_organization, +) from onadata.libs.utils.api_export_tools import custom_response_handler from onadata.libs.utils.cache_tools import ( - PROJ_BASE_FORMS_CACHE, PROJ_FORMS_CACHE, PROJ_NUM_DATASET_CACHE, - PROJ_OWNER_CACHE, PROJ_SUB_DATE_CACHE, reset_project_cache, safe_delete) + PROJ_BASE_FORMS_CACHE, + PROJ_FORMS_CACHE, + PROJ_NUM_DATASET_CACHE, + PROJ_OWNER_CACHE, + PROJ_SUB_DATE_CACHE, + reset_project_cache, + safe_delete, +) from onadata.libs.utils.common_tags import MEMBERS, XFORM_META_PERMS -from onadata.libs.utils.logger_tools import (publish_form, - response_with_mimetype_and_name) -from onadata.libs.utils.project_utils import (set_project_perms_to_xform, - set_project_perms_to_xform_async) -from onadata.libs.utils.user_auth import (check_and_set_form_by_id, - check_and_set_form_by_id_string) +from onadata.libs.utils.logger_tools import ( + publish_form, + response_with_mimetype_and_name, +) +from onadata.libs.utils.project_utils import ( + set_project_perms_to_xform, + set_project_perms_to_xform_async, +) +from onadata.libs.utils.user_auth import ( + check_and_set_form_by_id, + check_and_set_form_by_id_string, +) DECIMAL_PRECISION = 2 +# pylint: disable=invalid-name +User = get_user_model() + def _get_first_last_names(name): name_split = name.split() first_name = name_split[0] - last_name = u'' + last_name = "" if len(name_split) > 1: - last_name = u' '.join(name_split[1:]) + last_name = " ".join(name_split[1:]) return first_name, last_name def _get_id_for_type(record, mongo_field): date_field = datetime_from_str(record[mongo_field]) - mongo_str = '$' + mongo_field + mongo_str = "$" + mongo_field - return {"$substr": [mongo_str, 0, 10]} if isinstance(date_field, datetime)\ + return ( + {"$substr": [mongo_str, 0, 10]} + if isinstance(date_field, datetime) else mongo_str + ) def get_accessible_forms(owner=None, shared_form=False, shared_data=False): @@ -88,13 +118,14 @@ def get_accessible_forms(owner=None, shared_form=False, shared_data=False): if shared_form and not shared_data: xforms = xforms.filter(shared=True) - elif (shared_form and shared_data) or \ - (owner == 'public' and not shared_form and not shared_data): + elif (shared_form and shared_data) or ( + owner == "public" and not shared_form and not shared_data + ): xforms = xforms.filter(Q(shared=True) | Q(shared_data=True)) elif not shared_form and shared_data: xforms = xforms.filter(shared_data=True) - if owner != 'public': + if owner != "public": xforms = xforms.filter(user__username=owner) return xforms.distinct() @@ -109,33 +140,29 @@ def create_organization(name, creator): """ organization, _created = User.objects.get_or_create(username__iexact=name) organization_profile = OrganizationProfile.objects.create( - user=organization, creator=creator) + user=organization, creator=creator + ) return organization_profile def create_organization_object(org_name, creator, attrs=None): - '''Creates an OrganizationProfile object without saving to the database''' + """Creates an OrganizationProfile object without saving to the database""" attrs = attrs if attrs else {} - name = attrs.get('name', org_name) if attrs else org_name + name = attrs.get("name", org_name) if attrs else org_name first_name, last_name = _get_first_last_names(name) - email = attrs.get('email', u'') if attrs else u'' + email = attrs.get("email", "") if attrs else "" new_user = User( username=org_name, first_name=first_name, last_name=last_name, email=email, - is_active=getattr( - settings, - 'ORG_ON_CREATE_IS_ACTIVE', - True)) + is_active=getattr(settings, "ORG_ON_CREATE_IS_ACTIVE", True), + ) new_user.save() try: - registration_profile = RegistrationProfile.objects.create_profile( - new_user) - except IntegrityError: - raise ValidationError(_( - u"%s already exists" % org_name - )) + registration_profile = RegistrationProfile.objects.create_profile(new_user) + except IntegrityError as e: + raise ValidationError(_(f"{org_name} already exists")) from e if email: site = Site.objects.get(pk=settings.SITE_ID) registration_profile.send_activation_email(site) @@ -144,11 +171,12 @@ def create_organization_object(org_name, creator, attrs=None): name=name, creator=creator, created_by=creator, - city=attrs.get('city', u''), - country=attrs.get('country', u''), - organization=attrs.get('organization', u''), - home_page=attrs.get('home_page', u''), - twitter=attrs.get('twitter', u'')) + city=attrs.get("city", ""), + country=attrs.get("country", ""), + organization=attrs.get("organization", ""), + home_page=attrs.get("home_page", ""), + twitter=attrs.get("twitter", ""), + ) return profile @@ -157,15 +185,18 @@ def create_organization_team(organization, name, permission_names=None): Creates an organization team with the given permissions as defined in permission_names. """ - organization = organization.user \ - if isinstance(organization, OrganizationProfile) else organization + organization = ( + organization.user + if isinstance(organization, OrganizationProfile) + else organization + ) team = Team.objects.create(organization=organization, name=name) - content_type = ContentType.objects.get( - app_label='api', model='organizationprofile') + content_type = ContentType.objects.get(app_label="api", model="organizationprofile") if permission_names: # get permission objects perms = Permission.objects.filter( - codename__in=permission_names, content_type=content_type) + codename__in=permission_names, content_type=content_type + ) if perms: team.permissions.add(*tuple(perms)) return team @@ -176,8 +207,7 @@ def get_organization_members_team(organization): create members team if it does not exist and add organization owner to the members team""" try: - team = Team.objects.get(name=u'%s#%s' % (organization.user.username, - MEMBERS)) + team = Team.objects.get(name=f"{organization.user.username}#{MEMBERS}") except Team.DoesNotExist: team = create_organization_team(organization, MEMBERS) add_user_to_team(team, organization.user) @@ -192,14 +222,12 @@ def get_or_create_organization_owners_team(org): :param org: organization :return: Owners team of the organization """ - team_name = f'{org.user.username}#{Team.OWNER_TEAM_NAME}' + team_name = f"{org.user.username}#{Team.OWNER_TEAM_NAME}" try: team = Team.objects.get(name=team_name, organization=org.user) except Team.DoesNotExist: - from multidb.pinning import use_master # pylint: disable=import-error with use_master: - queryset = Team.objects.filter( - name=team_name, organization=org.user) + queryset = Team.objects.filter(name=team_name, organization=org.user) if queryset.count() > 0: return queryset.first() # pylint: disable=no-member return create_owner_team_and_assign_permissions(org) @@ -234,12 +262,11 @@ def remove_user_from_team(team, user): user.groups.remove(team) # remove the permission - remove_perm('view_team', user, team) + remove_perm("view_team", user, team) # if team is owners team remove more perms if team.name.find(Team.OWNER_TEAM_NAME) > 0: - owners_team = get_or_create_organization_owners_team( - team.organization.profile) + owners_team = get_or_create_organization_owners_team(team.organization.profile) members_team = get_organization_members_team(team.organization.profile) for perm in get_perms_for_model(Team): remove_perm(perm.codename, user, owners_team) @@ -260,7 +287,7 @@ def add_user_to_team(team, user): user.groups.add(team) # give the user perms to view the team - assign_perm('view_team', user, team) + assign_perm("view_team", user, team) # if team is owners team assign more perms if team.name.find(Team.OWNER_TEAM_NAME) > 0: @@ -293,10 +320,8 @@ def _get_owners(organization): return [ user - for user in get_or_create_organization_owners_team( - organization).user_set.all() - if get_role_in_org(user, organization) == 'owner' - and organization.user != user + for user in get_or_create_organization_owners_team(organization).user_set.all() + if get_role_in_org(user, organization) == "owner" and organization.user != user ] @@ -318,7 +343,8 @@ def create_organization_project(organization, project_name, created_by): name=project_name, organization=organization, created_by=created_by, - metadata='{}') + metadata="{}", + ) return project @@ -343,7 +369,8 @@ def publish_xlsform(request, owner, id_string=None, project=None): Publishes XLSForm & creates an XFormVersion object given a request. """ survey = do_publish_xlsform( - request.user, request.data, request.FILES, owner, id_string, project) + request.user, request.data, request.FILES, owner, id_string, project + ) return survey @@ -354,18 +381,19 @@ def do_publish_xlsform(user, post, files, owner, id_string=None, project=None): """ if id_string and project: xform = get_object_or_404( - XForm, user=owner, id_string=id_string, project=project) + XForm, user=owner, id_string=id_string, project=project + ) if not ManagerRole.user_has_role(user, xform): raise exceptions.PermissionDenied( - _("{} has no manager/owner role to the form {}".format( - user, xform))) - elif not user.has_perm('can_add_xform', owner.profile): + _(f"{user} has no manager/owner role to the form {xform}") + ) + elif not user.has_perm("can_add_xform", owner.profile): raise exceptions.PermissionDenied( - detail=_(u"User %(user)s has no permission to add xforms to " - "account %(account)s" % { - 'user': user.username, - 'account': owner.username - })) + detail=_( + f"User {user.username} has no permission to add xforms to " + f"account {owner.username}" + ) + ) def set_form(): """ @@ -373,8 +401,8 @@ def set_form(): """ if project: - args = (post and dict(listitems(post))) or {} - args['project'] = project.pk + args = dict(list(iteritems(post))) if post else {} + args["project"] = project.pk else: args = post @@ -395,11 +423,11 @@ def set_form(): Instantiates QuickConverter form to publish a form. """ props = { - 'project': project.pk, - 'dropbox_xls_url': request.data.get('dropbox_xls_url'), - 'xls_url': request.data.get('xls_url'), - 'csv_url': request.data.get('csv_url'), - 'text_xls_form': request.data.get('text_xls_form') + "project": project.pk, + "dropbox_xls_url": request.data.get("dropbox_xls_url"), + "xls_url": request.data.get("xls_url"), + "csv_url": request.data.get("csv_url"), + "text_xls_form": request.data.get("text_xls_form"), } form = QuickConverter(props, request.FILES) @@ -414,30 +442,28 @@ def id_string_exists_in_account(): otherwise returns False. """ try: - XForm.objects.get( - user=project.organization, id_string=xform.id_string) + XForm.objects.get(user=project.organization, id_string=xform.id_string) except XForm.DoesNotExist: return False return True - if 'formid' in request.data: - xform = get_object_or_404(XForm, pk=request.data.get('formid')) - safe_delete('{}{}'.format(PROJ_OWNER_CACHE, xform.project.pk)) - safe_delete('{}{}'.format(PROJ_FORMS_CACHE, xform.project.pk)) - safe_delete('{}{}'.format(PROJ_BASE_FORMS_CACHE, xform.project.pk)) - safe_delete('{}{}'.format(PROJ_NUM_DATASET_CACHE, xform.project.pk)) - safe_delete('{}{}'.format(PROJ_SUB_DATE_CACHE, xform.project.pk)) + if "formid" in request.data: + xform = get_object_or_404(XForm, pk=request.data.get("formid")) + safe_delete(f"{PROJ_OWNER_CACHE}{xform.project.pk}") + safe_delete(f"{PROJ_FORMS_CACHE}{xform.project.pk}") + safe_delete(f"{PROJ_BASE_FORMS_CACHE}{xform.project.pk}") + safe_delete(f"{PROJ_NUM_DATASET_CACHE}{xform.project.pk}") + safe_delete(f"{PROJ_SUB_DATE_CACHE}{xform.project.pk}") if not ManagerRole.user_has_role(request.user, xform): raise exceptions.PermissionDenied( - _("{} has no manager/owner role to the form {}".format( - request.user, xform))) + _(f"{request.user} has no manager/owner role to the form {xform}") + ) - msg = 'Form with the same id_string already exists in this account' + msg = "Form with the same id_string already exists in this account" # Without this check, a user can't transfer a form to projects that # he/she owns because `id_string_exists_in_account` will always # return true - if project.organization != xform.user and \ - id_string_exists_in_account(): + if project.organization != xform.user and id_string_exists_in_account(): raise exceptions.ParseError(_(msg)) xform.user = project.organization xform.project = project @@ -445,8 +471,8 @@ def id_string_exists_in_account(): try: with transaction.atomic(): xform.save() - except IntegrityError: - raise exceptions.ParseError(_(msg)) + except IntegrityError as e: + raise exceptions.ParseError(_(msg)) from e else: # First assign permissions to the person who uploaded the form OwnerRole.add(request.user, xform) @@ -483,7 +509,8 @@ def get_xform(formid, request, username=None): if not xform: raise exceptions.PermissionDenied( - _("You do not have permission to view data from this form.")) + _("You do not have permission to view data from this form.") + ) return xform @@ -529,12 +556,13 @@ class TagForm(forms.Form): """ Simple TagForm class to validate tags in a request. """ + tags = TagField() form = TagForm(request.data) if form.is_valid(): - tags = form.cleaned_data.get('tags', None) + tags = form.cleaned_data.get("tags", None) if tags: for tag in tags: @@ -559,9 +587,9 @@ def get_data_value_objects(value): Looks for 'dataview 123 fruits.csv' or 'xform 345 fruits.csv'. """ model = None - if value.startswith('dataview'): + if value.startswith("dataview"): model = DataView - elif value.startswith('xform'): + elif value.startswith("xform"): model = XForm if model: @@ -575,8 +603,8 @@ def get_data_value_objects(value): if metadata.data_file: file_path = metadata.data_file.name - filename, extension = os.path.splitext(file_path.split('/')[-1]) - extension = extension.strip('.') + filename, extension = os.path.splitext(file_path.split("/")[-1]) + extension = extension.strip(".") dfs = get_storage_class()() if dfs.exists(file_path): @@ -586,7 +614,8 @@ def get_data_value_objects(value): extension=extension, show_date=False, file_path=file_path, - full_mime=True) + full_mime=True, + ) return response return HttpResponseNotFound() @@ -600,10 +629,12 @@ def get_data_value_objects(value): return custom_response_handler( request, - xform, {}, + xform, + {}, Export.CSV_EXPORT, filename=filename, - dataview=dataview) + dataview=dataview, + ) return HttpResponseRedirect(metadata.data_value) @@ -615,7 +646,7 @@ def check_inherit_permission_from_project(xform_id, user): if there is a difference applies the project permissions to the user for the given xform_id. """ - if xform_id == 'public': + if xform_id == "public": return try: @@ -624,8 +655,12 @@ def check_inherit_permission_from_project(xform_id, user): return # get the project_xform - xform = XForm.objects.filter(pk=xform_id).select_related('project').only( - 'project_id', 'id').first() + xform = ( + XForm.objects.filter(pk=xform_id) + .select_related("project") + .only("project_id", "id") + .first() + ) if not xform: return @@ -656,8 +691,11 @@ def get_baseviewset_class(): the default in onadata :return: the default baseviewset """ - return import_string(settings.BASE_VIEWSET) \ - if settings.BASE_VIEWSET else DefaultBaseViewset + return ( + import_string(settings.BASE_VIEWSET) + if settings.BASE_VIEWSET + else DefaultBaseViewset + ) def generate_tmp_path(uploaded_csv_file): @@ -668,9 +706,9 @@ def generate_tmp_path(uploaded_csv_file): """ if isinstance(uploaded_csv_file, InMemoryUploadedFile): uploaded_csv_file.open() - tmp_file = tempfile.NamedTemporaryFile(delete=False) - tmp_file.write(uploaded_csv_file.read()) - tmp_path = tmp_file.name + with tempfile.NamedTemporaryFile(delete=False) as tmp_file: + tmp_file.write(uploaded_csv_file.read()) + tmp_path = tmp_file.name uploaded_csv_file.close() else: tmp_path = uploaded_csv_file.temporary_file_path() @@ -694,31 +732,31 @@ def get_xform_users(xform): org_members = get_team_members(user.username) data[user] = { - 'permissions': [], - 'is_org': is_organization(user.profile), - 'metadata': user.profile.metadata, - 'first_name': user.first_name, - 'last_name': user.last_name, - 'user': user.username + "permissions": [], + "is_org": is_organization(user.profile), + "metadata": user.profile.metadata, + "first_name": user.first_name, + "last_name": user.last_name, + "user": user.username, } if perm.user in data: - data[perm.user]['permissions'].append(perm.permission.codename) + data[perm.user]["permissions"].append(perm.permission.codename) for user in org_members: if user not in data: data[user] = { - 'permissions': get_perms(user, xform), - 'is_org': is_organization(user.profile), - 'metadata': user.profile.metadata, - 'first_name': user.first_name, - 'last_name': user.last_name, - 'user': user.username + "permissions": get_perms(user, xform), + "is_org": is_organization(user.profile), + "metadata": user.profile.metadata, + "first_name": user.first_name, + "last_name": user.last_name, + "user": user.username, } - for k in data: - data[k]['permissions'].sort() - data[k]['role'] = get_role(data[k]['permissions'], xform) - del data[k]['permissions'] + for value in data.values(): + value["permissions"].sort() + value["role"] = get_role(value["permissions"], xform) + del value["permissions"] return data @@ -731,8 +769,7 @@ def get_team_members(org_username): """ members = [] try: - team = Team.objects.get( - name="{}#{}".format(org_username, MEMBERS)) + team = Team.objects.get(name=f"{org_username}#{MEMBERS}") except Team.DoesNotExist: pass else: @@ -750,20 +787,18 @@ def update_role_by_meta_xform_perms(xform): editor_role_list = [EditorRole, EditorMinorRole] editor_role = {role.name: role for role in editor_role_list} - dataentry_role_list = [ - DataEntryMinorRole, DataEntryOnlyRole, DataEntryRole - ] + dataentry_role_list = [DataEntryMinorRole, DataEntryOnlyRole, DataEntryRole] dataentry_role = {role.name: role for role in dataentry_role_list} if metadata: - meta_perms = metadata.data_value.split('|') + meta_perms = metadata.data_value.split("|") # update roles users = get_xform_users(xform) for user in users: - role = users.get(user).get('role') + role = users.get(user).get("role") if role in editor_role: role = ROLES.get(meta_perms[0]) role.add(user, xform) @@ -774,15 +809,17 @@ def update_role_by_meta_xform_perms(xform): def replace_attachment_name_with_url(data): + """Replaces the attachment filename with a URL in ``data`` object.""" site_url = Site.objects.get_current().domain for record in data: - attachments: dict = record.json.get('_attachments') + attachments: dict = record.json.get("_attachments") if attachments: attachment_details = [ - (attachment['name'], attachment['download_url']) + (attachment["name"], attachment["download_url"]) for attachment in attachments - if 'download_url' in attachment] + if "download_url" in attachment + ] question_keys = list(record.json.keys()) question_values = list(record.json.values()) diff --git a/onadata/apps/api/urls/v1_urls.py b/onadata/apps/api/urls/v1_urls.py index 631fb96937..15d3a830a9 100644 --- a/onadata/apps/api/urls/v1_urls.py +++ b/onadata/apps/api/urls/v1_urls.py @@ -2,7 +2,7 @@ """ Custom rest_framework Router - MultiLookupRouter. """ -from django.conf.urls import url +from django.urls import re_path from django.contrib import admin from rest_framework import routers from rest_framework.urlpatterns import format_suffix_patterns @@ -107,7 +107,7 @@ def get_urls(self): }) view = viewset.as_view(mapping, **initkwargs) name = route.name.format(basename=basename) - extra_urls.append(url(regex, view, name=name)) + extra_urls.append(re_path(regex, view, name=name)) if self.include_format_suffixes: extra_urls = format_suffix_patterns(extra_urls) diff --git a/onadata/apps/api/viewsets/attachment_viewset.py b/onadata/apps/api/viewsets/attachment_viewset.py index af38671c40..ac5af6a90f 100644 --- a/onadata/apps/api/viewsets/attachment_viewset.py +++ b/onadata/apps/api/viewsets/attachment_viewset.py @@ -1,7 +1,7 @@ from builtins import str as text from django.http import Http404 -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.core.files.storage import default_storage from django.conf import settings from rest_framework import renderers diff --git a/onadata/apps/api/viewsets/briefcase_viewset.py b/onadata/apps/api/viewsets/briefcase_viewset.py index 79522e821a..b6fa26753c 100644 --- a/onadata/apps/api/viewsets/briefcase_viewset.py +++ b/onadata/apps/api/viewsets/briefcase_viewset.py @@ -1,24 +1,25 @@ +# -*- coding: utf-8 -*- +""" +The /briefcase API implementation. +""" from xml.dom import NotFoundErr + from django.conf import settings +from django.contrib.auth import get_user_model from django.core.files import File from django.core.validators import ValidationError -from django.contrib.auth.models import User from django.http import Http404 -from django.utils.translation import ugettext as _ -from django.utils import six - -from rest_framework import exceptions -from rest_framework import mixins -from rest_framework import status -from rest_framework import viewsets -from rest_framework import permissions +from django.utils.translation import gettext as _ + +import six +from rest_framework import exceptions, mixins, permissions, status, viewsets from rest_framework.decorators import action from rest_framework.generics import get_object_or_404 from rest_framework.renderers import BrowsableAPIRenderer from rest_framework.response import Response -from onadata.apps.api.tools import get_media_file_response from onadata.apps.api.permissions import ViewDjangoObjectPermissions +from onadata.apps.api.tools import get_media_file_response from onadata.apps.logger.models.attachment import Attachment from onadata.apps.logger.models.instance import Instance from onadata.apps.logger.models.xform import XForm @@ -29,22 +30,26 @@ from onadata.libs.authentication import DigestAuthentication from onadata.libs.mixins.openrosa_headers_mixin import get_openrosa_headers from onadata.libs.renderers.renderers import TemplateXMLRenderer -from onadata.libs.serializers.xform_serializer import XFormListSerializer -from onadata.libs.serializers.xform_serializer import XFormManifestSerializer -from onadata.libs.utils.logger_tools import publish_form -from onadata.libs.utils.logger_tools import PublishXForm +from onadata.libs.serializers.xform_serializer import ( + XFormListSerializer, + XFormManifestSerializer, +) +from onadata.libs.utils.logger_tools import PublishXForm, publish_form from onadata.libs.utils.viewer_tools import get_form +# pylint: disable=invalid-name +User = get_user_model() + def _extract_uuid(text): if isinstance(text, six.string_types): - form_id_parts = text.split('/') + form_id_parts = text.split("/") if form_id_parts.__len__() < 2: - raise ValidationError(_(u"Invalid formId %s." % text)) + raise ValidationError(_(f"Invalid formId {text}.")) text = form_id_parts[1] - text = text[text.find("@key="):-1].replace("@key=", "") + text = text[text.find("@key=") : -1].replace("@key=", "") if text.startswith("uuid:"): text = text.replace("uuid:", "") @@ -52,59 +57,68 @@ def _extract_uuid(text): return text -def _extract_id_string(formId): - if isinstance(formId, six.string_types): - return formId[0:formId.find('[')] +def _extract_id_string(id_string): + if isinstance(id_string, six.string_types): + return id_string[0 : id_string.find("[")] - return formId + return id_string def _parse_int(num): try: return num and int(num) except ValueError: - pass + return None -class BriefcaseViewset(mixins.CreateModelMixin, - mixins.RetrieveModelMixin, mixins.ListModelMixin, - viewsets.GenericViewSet): +# pylint: disable=too-many-ancestors +class BriefcaseViewset( + mixins.CreateModelMixin, + mixins.RetrieveModelMixin, + mixins.ListModelMixin, + viewsets.GenericViewSet, +): """ Implements the [Briefcase Aggregate API](\ https://code.google.com/p/opendatakit/wiki/BriefcaseAggregateAPI). """ + authentication_classes = (DigestAuthentication,) filter_backends = (filters.AnonDjangoObjectPermissionFilter,) queryset = XForm.objects.all() - permission_classes = (permissions.IsAuthenticated, - ViewDjangoObjectPermissions) + permission_classes = (permissions.IsAuthenticated, ViewDjangoObjectPermissions) renderer_classes = (TemplateXMLRenderer, BrowsableAPIRenderer) serializer_class = XFormListSerializer - template_name = 'openrosa_response.xml' + template_name = "openrosa_response.xml" + # pylint: disable=unused-argument def get_object(self, queryset=None): - formId = self.request.GET.get('formId', '') - id_string = _extract_id_string(formId) - uuid = _extract_uuid(formId) - username = self.kwargs.get('username') - - obj = get_object_or_404(Instance, - xform__user__username__iexact=username, - xform__id_string__iexact=id_string, - uuid=uuid) + """Returns an Instance submission object for the given UUID.""" + form_id = self.request.GET.get("formId", "") + id_string = _extract_id_string(form_id) + uuid = _extract_uuid(form_id) + username = self.kwargs.get("username") + + obj = get_object_or_404( + Instance, + xform__user__username__iexact=username, + xform__id_string__iexact=id_string, + uuid=uuid, + ) self.check_object_permissions(self.request, obj.xform) return obj + # pylint: disable=too-many-branches def filter_queryset(self, queryset): - username = self.kwargs.get('username') + """Filters an XForm submission instances using ODK Aggregate query parameters.""" + username = self.kwargs.get("username") if username is None and self.request.user.is_anonymous: # raises a permission denied exception, forces authentication self.permission_denied(self.request) if username is not None and self.request.user.is_anonymous: - profile = get_object_or_404( - UserProfile, user__username__iexact=username) + profile = get_object_or_404(UserProfile, user__username__iexact=username) if profile.require_auth: # raises a permission denied exception, forces authentication @@ -112,29 +126,26 @@ def filter_queryset(self, queryset): else: queryset = queryset.filter(user=profile.user) else: - queryset = super(BriefcaseViewset, self).filter_queryset(queryset) + queryset = super().filter_queryset(queryset) - formId = self.request.GET.get('formId', '') + id_string = self.request.GET.get("formId", "") - if formId.find('[') != -1: - formId = _extract_id_string(formId) + if id_string.find("[") != -1: + id_string = _extract_id_string(id_string) - xform_kwargs = { - 'queryset': queryset, - 'id_string__iexact': formId - } + xform_kwargs = {"queryset": queryset, "id_string__iexact": id_string} if username: - xform_kwargs['user__username__iexact'] = username + xform_kwargs["user__username__iexact"] = username xform = get_form(xform_kwargs) self.check_object_permissions(self.request, xform) instances = Instance.objects.filter( - xform=xform, deleted_at__isnull=True).values( - 'pk', 'uuid') + xform=xform, deleted_at__isnull=True + ).values("pk", "uuid") if xform.encrypted: instances = instances.filter(media_all_received=True) - instances = instances.order_by('pk') - num_entries = self.request.GET.get('numEntries') - cursor = self.request.GET.get('cursor') + instances = instances.order_by("pk") + num_entries = self.request.GET.get("numEntries") + cursor = self.request.GET.get("cursor") cursor = _parse_int(cursor) if cursor: @@ -150,118 +161,139 @@ def filter_queryset(self, queryset): # and removes the need to perform a count on the database. instance_count = len(instances) + # pylint: disable=attribute-defined-outside-init if instance_count > 0: last_instance = instances[instance_count - 1] - self.resumptionCursor = last_instance.get('pk') + self.resumption_cursor = last_instance.get("pk") elif instance_count == 0 and cursor: - self.resumptionCursor = cursor + self.resumption_cursor = cursor else: - self.resumptionCursor = 0 + self.resumption_cursor = 0 return instances def create(self, request, *args, **kwargs): - if request.method.upper() == 'HEAD': - return Response(status=status.HTTP_204_NO_CONTENT, - headers=get_openrosa_headers(request), - template_name=self.template_name) - - xform_def = request.FILES.get('form_def_file', None) + """Accepts an XForm XML and publishes it as a form.""" + if request.method.upper() == "HEAD": + return Response( + status=status.HTTP_204_NO_CONTENT, + headers=get_openrosa_headers(request), + template_name=self.template_name, + ) + + xform_def = request.FILES.get("form_def_file", None) response_status = status.HTTP_201_CREATED - username = kwargs.get('username') - form_user = (username and get_object_or_404(User, username=username)) \ - or request.user + username = kwargs.get("username") + form_user = ( + get_object_or_404(User, username=username) if username else request.user + ) - if not request.user.has_perm('can_add_xform', form_user.profile): + if not request.user.has_perm("can_add_xform", form_user.profile): raise exceptions.PermissionDenied( - detail=_(u"User %(user)s has no permission to add xforms to " - "account %(account)s" % - {'user': request.user.username, - 'account': form_user.username})) + detail=_( + "User %(user)s has no permission to add xforms to " + "account %(account)s" + % {"user": request.user.username, "account": form_user.username} + ) + ) data = {} if isinstance(xform_def, File): do_form_upload = PublishXForm(xform_def, form_user) - dd = publish_form(do_form_upload.publish_xform) + data_dictionary = publish_form(do_form_upload.publish_xform) - if isinstance(dd, XForm): - data['message'] = _( - u"%s successfully published." % dd.id_string) + if isinstance(data_dictionary, XForm): + data["message"] = _( + f"{data_dictionary.id_string} successfully published." + ) else: - data['message'] = dd['text'] + data["message"] = data_dictionary["text"] response_status = status.HTTP_400_BAD_REQUEST else: - data['message'] = _(u"Missing xml file.") + data["message"] = _("Missing xml file.") response_status = status.HTTP_400_BAD_REQUEST - return Response(data, status=response_status, - headers=get_openrosa_headers(request, - location=False), - template_name=self.template_name) + return Response( + data, + status=response_status, + headers=get_openrosa_headers(request, location=False), + template_name=self.template_name, + ) def list(self, request, *args, **kwargs): + """Returns a list of submissions with reference submission download.""" + # pylint: disable=attribute-defined-outside-init self.object_list = self.filter_queryset(self.get_queryset()) - data = {'instances': self.object_list, - 'resumptionCursor': self.resumptionCursor} + data = { + "instances": self.object_list, + "resumptionCursor": self.resumption_cursor, + } - return Response(data, - headers=get_openrosa_headers(request, - location=False), - template_name='submissionList.xml') + return Response( + data, + headers=get_openrosa_headers(request, location=False), + template_name="submissionList.xml", + ) def retrieve(self, request, *args, **kwargs): + """Returns a single submission XML for download.""" + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() xml_obj = clean_and_parse_xml(self.object.xml) submission_xml_root_node = xml_obj.documentElement + submission_xml_root_node.setAttribute("instanceID", f"uuid:{self.object.uuid}") submission_xml_root_node.setAttribute( - 'instanceID', u'uuid:%s' % self.object.uuid) - submission_xml_root_node.setAttribute( - 'submissionDate', self.object.date_created.isoformat() + "submissionDate", self.object.date_created.isoformat() ) if getattr(settings, "SUPPORT_BRIEFCASE_SUBMISSION_DATE", True): # Remove namespace attribute if any try: - submission_xml_root_node.removeAttribute('xmlns') + submission_xml_root_node.removeAttribute("xmlns") except NotFoundErr: pass data = { - 'submission_data': submission_xml_root_node.toxml(), - 'media_files': Attachment.objects.filter(instance=self.object), - 'host': request.build_absolute_uri().replace( - request.get_full_path(), '') + "submission_data": submission_xml_root_node.toxml(), + "media_files": Attachment.objects.filter(instance=self.object), + "host": request.build_absolute_uri().replace(request.get_full_path(), ""), } return Response( data, headers=get_openrosa_headers(request, location=False), - template_name='downloadSubmission.xml' + template_name="downloadSubmission.xml", ) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def manifest(self, request, *args, **kwargs): + """Returns list of media content.""" + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() - object_list = MetaData.objects.filter(data_type='media', - object_id=self.object.id) + object_list = MetaData.objects.filter( + data_type="media", object_id=self.object.id + ) context = self.get_serializer_context() - serializer = XFormManifestSerializer(object_list, many=True, - context=context) + serializer = XFormManifestSerializer(object_list, many=True, context=context) - return Response(serializer.data, - headers=get_openrosa_headers(request, location=False)) + return Response( + serializer.data, headers=get_openrosa_headers(request, location=False) + ) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def media(self, request, *args, **kwargs): + """Returns a single media content.""" + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() - pk = kwargs.get('metadata') + metadata_pk = kwargs.get("metadata") - if not pk: + if not metadata_pk: raise Http404() meta_obj = get_object_or_404( - MetaData, data_type='media', xform=self.object, pk=pk) + MetaData, data_type="media", xform=self.object, pk=metadata_pk + ) return get_media_file_response(meta_obj) diff --git a/onadata/apps/api/viewsets/connect_viewset.py b/onadata/apps/api/viewsets/connect_viewset.py index d2d88bfb2f..a1c0f3817f 100644 --- a/onadata/apps/api/viewsets/connect_viewset.py +++ b/onadata/apps/api/viewsets/connect_viewset.py @@ -1,7 +1,7 @@ from django.core.exceptions import MultipleObjectsReturned from django.utils import timezone from django.utils.decorators import classonlymethod -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.views.decorators.cache import never_cache from rest_framework import status, viewsets from rest_framework.authtoken.models import Token diff --git a/onadata/apps/api/viewsets/data_viewset.py b/onadata/apps/api/viewsets/data_viewset.py index 918b7a6980..0919761ad7 100644 --- a/onadata/apps/api/viewsets/data_viewset.py +++ b/onadata/apps/api/viewsets/data_viewset.py @@ -1,6 +1,10 @@ +# -*- coding: utf-8 -*- +""" +The /data API endpoint. +""" import json +import math import types -from builtins import str as text from typing import Union from django.conf import settings @@ -8,12 +12,11 @@ from django.db.models import Q from django.db.models.query import QuerySet from django.db.utils import DataError, OperationalError -from django.http import Http404 -from django.http import StreamingHttpResponse -from django.utils import six +from django.http import Http404, StreamingHttpResponse from django.utils import timezone -from distutils.util import strtobool -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ + +import six from rest_framework import status from rest_framework.decorators import action from rest_framework.exceptions import ParseError @@ -22,61 +25,67 @@ from rest_framework.settings import api_settings from rest_framework.viewsets import ModelViewSet -from onadata.apps.api.permissions import ConnectViewsetPermissions -from onadata.apps.api.permissions import XFormPermissions -from onadata.apps.api.tools import add_tags_to_instance -from onadata.apps.api.tools import get_baseviewset_class -from onadata.apps.logger.models import OsmData, MergedXForm +from onadata.apps.api.permissions import ConnectViewsetPermissions, XFormPermissions +from onadata.apps.api.tools import add_tags_to_instance, get_baseviewset_class +from onadata.apps.logger.models import MergedXForm, OsmData from onadata.apps.logger.models.attachment import Attachment -from onadata.apps.logger.models.instance import ( - Instance, - FormInactiveError) +from onadata.apps.logger.models.instance import FormInactiveError, Instance from onadata.apps.logger.models.xform import XForm -from onadata.apps.messaging.constants import XFORM, SUBMISSION_DELETED +from onadata.apps.messaging.constants import SUBMISSION_DELETED, XFORM from onadata.apps.messaging.serializers import send_message -from onadata.apps.viewer.models.parsed_instance import get_etag_hash_from_query -from onadata.apps.viewer.models.parsed_instance import get_sql_with_params -from onadata.apps.viewer.models.parsed_instance import get_where_clause -from onadata.apps.viewer.models.parsed_instance import query_data +from onadata.apps.viewer.models.parsed_instance import ( + get_etag_hash_from_query, + get_sql_with_params, + get_where_clause, + query_data, +) from onadata.libs import filters -from onadata.libs.data import parse_int -from onadata.libs.exceptions import EnketoError -from onadata.libs.exceptions import NoRecordsPermission +from onadata.libs.data import parse_int, strtobool +from onadata.libs.exceptions import EnketoError, NoRecordsPermission from onadata.libs.mixins.anonymous_user_public_forms_mixin import ( - AnonymousUserPublicFormsMixin) -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin + AnonymousUserPublicFormsMixin, +) +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.pagination import CountOverridablePageNumberPagination -from onadata.libs.permissions import CAN_DELETE_SUBMISSION, \ - filter_queryset_xform_meta_perms, filter_queryset_xform_meta_perms_sql +from onadata.libs.permissions import ( + CAN_DELETE_SUBMISSION, + filter_queryset_xform_meta_perms, + filter_queryset_xform_meta_perms_sql, +) from onadata.libs.renderers import renderers from onadata.libs.serializers.data_serializer import ( - DataInstanceSerializer, DataInstanceXMLSerializer, - InstanceHistorySerializer) -from onadata.libs.serializers.data_serializer import DataSerializer -from onadata.libs.serializers.data_serializer import JsonDataSerializer -from onadata.libs.serializers.data_serializer import OSMSerializer + DataInstanceSerializer, + DataInstanceXMLSerializer, + DataSerializer, + InstanceHistorySerializer, + JsonDataSerializer, + OSMSerializer, +) from onadata.libs.serializers.geojson_serializer import GeoJsonSerializer from onadata.libs.utils.api_export_tools import custom_response_handler from onadata.libs.utils.common_tools import json_stream -from onadata.libs.utils.viewer_tools import get_form_url, get_enketo_urls +from onadata.libs.utils.viewer_tools import get_enketo_urls, get_form_url -SAFE_METHODS = ['GET', 'HEAD', 'OPTIONS'] -SUBMISSION_RETRIEVAL_THRESHOLD = getattr(settings, - "SUBMISSION_RETRIEVAL_THRESHOLD", - 10000) +SAFE_METHODS = ["GET", "HEAD", "OPTIONS"] +SUBMISSION_RETRIEVAL_THRESHOLD = getattr( + settings, "SUBMISSION_RETRIEVAL_THRESHOLD", 10000 +) +# pylint: disable=invalid-name BaseViewset = get_baseviewset_class() def get_data_and_form(kwargs): - data_id = text(kwargs.get('dataid')) + """ + Checks if the dataid in ``kwargs`` is a valid integer. + """ + data_id = str(kwargs.get("dataid")) if not data_id.isdigit(): - raise ParseError(_(u"Data ID should be an integer")) + raise ParseError(_("Data ID should be an integer")) - return (data_id, kwargs.get('format')) + return (data_id, kwargs.get("format")) def delete_instance(instance, user): @@ -90,14 +99,18 @@ def delete_instance(instance, user): try: instance.set_deleted(timezone.now(), user) except FormInactiveError as e: - raise ParseError(text(e)) - - -class DataViewSet(AnonymousUserPublicFormsMixin, - AuthenticateHeaderMixin, - ETagsMixin, CacheControlMixin, - BaseViewset, - ModelViewSet): + raise ParseError(str(e)) from e + + +# pylint: disable=too-many-ancestors +class DataViewSet( + AnonymousUserPublicFormsMixin, + AuthenticateHeaderMixin, + ETagsMixin, + CacheControlMixin, + BaseViewset, + ModelViewSet, +): """ This endpoint provides access to submitted data. """ @@ -116,167 +129,192 @@ class DataViewSet(AnonymousUserPublicFormsMixin, renderers.FLOIPRenderer, ] - filter_backends = (filters.AnonDjangoObjectPermissionFilter, - filters.XFormOwnerFilter, - filters.DataFilter) + filter_backends = ( + filters.AnonDjangoObjectPermissionFilter, + filters.XFormOwnerFilter, + filters.DataFilter, + ) serializer_class = DataSerializer permission_classes = (XFormPermissions,) - lookup_field = 'pk' - lookup_fields = ('pk', 'dataid') + lookup_field = "pk" + lookup_fields = ("pk", "dataid") extra_lookup_fields = None data_count = None - public_data_endpoint = 'public' + public_data_endpoint = "public" pagination_class = CountOverridablePageNumberPagination queryset = XForm.objects.filter(deleted_at__isnull=True) def get_serializer_class(self): + """Returns appropriate serializer class based on context.""" pk_lookup, dataid_lookup = self.lookup_fields - pk = self.kwargs.get(pk_lookup) + form_pk = self.kwargs.get(pk_lookup) dataid = self.kwargs.get(dataid_lookup) - fmt = self.kwargs.get('format', self.request.GET.get("format")) + fmt = self.kwargs.get("format", self.request.GET.get("format")) sort = self.request.GET.get("sort") fields = self.request.GET.get("fields") if fmt == Attachment.OSM: serializer_class = OSMSerializer - elif fmt == 'geojson': + elif fmt == "geojson": serializer_class = GeoJsonSerializer - elif fmt == 'xml': + elif fmt == "xml": serializer_class = DataInstanceXMLSerializer - elif pk is not None and dataid is None \ - and pk != self.public_data_endpoint: + elif ( + form_pk is not None + and dataid is None + and form_pk != self.public_data_endpoint + ): if sort or fields: serializer_class = JsonDataSerializer else: serializer_class = DataInstanceSerializer else: - serializer_class = \ - super(DataViewSet, self).get_serializer_class() + serializer_class = super().get_serializer_class() return serializer_class + # pylint: disable=unused-argument def get_object(self, queryset=None): - obj = super(DataViewSet, self).get_object() + """Returns the appropriate object based on context.""" + obj = super().get_object() pk_lookup, dataid_lookup = self.lookup_fields - pk = self.kwargs.get(pk_lookup) + form_pk = self.kwargs.get(pk_lookup) dataid = self.kwargs.get(dataid_lookup) - if pk is not None and dataid is not None: + if form_pk is not None and dataid is not None: try: int(dataid) - except ValueError: - raise ParseError(_(u"Invalid dataid %(dataid)s" - % {'dataid': dataid})) + except ValueError as e: + raise ParseError(_(f"Invalid dataid {dataid}")) from e if not obj.is_merged_dataset: - obj = get_object_or_404(Instance, pk=dataid, xform__pk=pk, - deleted_at__isnull=True) + obj = get_object_or_404( + Instance, pk=dataid, xform__pk=form_pk, deleted_at__isnull=True + ) else: xforms = obj.mergedxform.xforms.filter(deleted_at__isnull=True) - pks = [xform_id - for xform_id in xforms.values_list('pk', flat=True)] + pks = list(xforms.values_list("pk", flat=True)) - obj = get_object_or_404(Instance, pk=dataid, xform_id__in=pks, - deleted_at__isnull=True) + obj = get_object_or_404( + Instance, pk=dataid, xform_id__in=pks, deleted_at__isnull=True + ) return obj def _get_public_forms_queryset(self): - return XForm.objects.filter(Q(shared=True) | Q(shared_data=True), - deleted_at__isnull=True) + return XForm.objects.filter( + Q(shared=True) | Q(shared_data=True), deleted_at__isnull=True + ) - def _filtered_or_shared_qs(self, qs, pk): - filter_kwargs = {self.lookup_field: pk} - qs = qs.filter(**filter_kwargs).only('id', 'shared') + def _filtered_or_shared_queryset(self, queryset, form_pk): + filter_kwargs = {self.lookup_field: form_pk} + queryset = queryset.filter(**filter_kwargs).only("id", "shared") - if not qs: - filter_kwargs['shared_data'] = True - qs = XForm.objects.filter(**filter_kwargs).only('id', 'shared') + if not queryset: + filter_kwargs["shared_data"] = True + queryset = XForm.objects.filter(**filter_kwargs).only("id", "shared") - if not qs: - raise Http404(_(u"No data matches with given query.")) + if not queryset: + raise Http404(_("No data matches with given query.")) - return qs + return queryset + # pylint: disable=unused-argument def filter_queryset(self, queryset, view=None): - qs = super(DataViewSet, self).filter_queryset( - queryset.only('id', 'shared')) - pk = self.kwargs.get(self.lookup_field) + """Returns and filters queryset based on context and query params.""" + queryset = super().filter_queryset(queryset.only("id", "shared")) + form_pk = self.kwargs.get(self.lookup_field) - if pk: + if form_pk: try: - int(pk) - except ValueError: - if pk == self.public_data_endpoint: - qs = self._get_public_forms_queryset() + int(form_pk) + except ValueError as e: + if form_pk == self.public_data_endpoint: + queryset = self._get_public_forms_queryset() else: - raise ParseError(_(u"Invalid pk %(pk)s" % {'pk': pk})) + raise ParseError(_(f"Invalid pk {form_pk}")) from e else: - qs = self._filtered_or_shared_qs(qs, pk) + queryset = self._filtered_or_shared_queryset(queryset, form_pk) else: - tags = self.request.query_params.get('tags') - not_tagged = self.request.query_params.get('not_tagged') + tags = self.request.query_params.get("tags") + not_tagged = self.request.query_params.get("not_tagged") if tags and isinstance(tags, six.string_types): - tags = tags.split(',') - qs = qs.filter(tags__name__in=tags) + tags = tags.split(",") + queryset = queryset.filter(tags__name__in=tags) if not_tagged and isinstance(not_tagged, six.string_types): - not_tagged = not_tagged.split(',') - qs = qs.exclude(tags__name__in=not_tagged) - - return qs - - @action(methods=['GET', 'POST', 'DELETE'], detail=True, - extra_lookup_fields=['label', ]) + not_tagged = not_tagged.split(",") + queryset = queryset.exclude(tags__name__in=not_tagged) + + return queryset + + @action( + methods=["GET", "POST", "DELETE"], + detail=True, + extra_lookup_fields=[ + "label", + ], + ) def labels(self, request, *args, **kwargs): + """ + Data labels API endpoint. + """ http_status = status.HTTP_400_BAD_REQUEST + # pylint: disable=attribute-defined-outside-init self.object = instance = self.get_object() - if request.method == 'POST': + if request.method == "POST": add_tags_to_instance(request, instance) http_status = status.HTTP_201_CREATED tags = instance.tags - label = kwargs.get('label') + label = kwargs.get("label") - if request.method == 'GET' and label: - data = [tag['name'] for tag in - tags.filter(name=label).values('name')] + if request.method == "GET" and label: + data = [tag["name"] for tag in tags.filter(name=label).values("name")] - elif request.method == 'DELETE' and label: + elif request.method == "DELETE" and label: count = tags.count() tags.remove(label) # Accepted, label does not exist hence nothing removed - http_status = status.HTTP_200_OK if count > tags.count() \ + http_status = ( + status.HTTP_200_OK + if count > tags.count() else status.HTTP_404_NOT_FOUND + ) data = list(tags.names()) else: data = list(tags.names()) - if request.method == 'GET': + if request.method == "GET": http_status = status.HTTP_200_OK - self.etag_data = data + setattr(self, "etag_data", data) return Response(data, status=http_status) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def enketo(self, request, *args, **kwargs): + """ + Data Enketo URLs endpoint + """ + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() data = {} if isinstance(self.object, XForm): - raise ParseError(_(u"Data id not provided.")) - elif(isinstance(self.object, Instance)): + raise ParseError(_("Data id not provided.")) + if isinstance(self.object, Instance): if request.user.has_perm("change_xform", self.object.xform): - return_url = request.query_params.get('return_url') + return_url = request.query_params.get("return_url") form_url = get_form_url( request, self.object.xform.user.username, - xform_pk=self.object.xform.id) + xform_pk=self.object.xform.id, + ) if not return_url: - raise ParseError(_(u"return_url not provided.")) + raise ParseError(_("return_url not provided.")) try: data = get_enketo_urls( @@ -284,141 +322,141 @@ def enketo(self, request, *args, **kwargs): self.object.xform.id_string, instance_id=self.object.uuid, instance_xml=self.object.xml, - return_url=return_url) + return_url=return_url, + ) if "edit_url" in data: data["url"] = data.pop("edit_url") except EnketoError as e: - raise ParseError(text(e)) + raise ParseError(str(e)) from e else: - raise PermissionDenied(_(u"You do not have edit permissions.")) + raise PermissionDenied(_("You do not have edit permissions.")) - self.etag_data = data + setattr(self, "etag_data", data) return Response(data=data) def destroy(self, request, *args, **kwargs): - instance_ids = request.data.get('instance_ids') - delete_all_submissions = strtobool( - request.data.get('delete_all', 'False')) + """Soft deletes submissions data.""" + instance_ids = request.data.get("instance_ids") + delete_all_submissions = strtobool(request.data.get("delete_all", "False")) + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() if isinstance(self.object, XForm): if not instance_ids and not delete_all_submissions: - raise ParseError(_(u"Data id(s) not provided.")) + raise ParseError(_("Data id(s) not provided.")) + initial_count = self.object.submission_count() + if delete_all_submissions: + # Update timestamp only for active records + self.object.instances.filter(deleted_at__isnull=True).update( + deleted_at=timezone.now(), + date_modified=timezone.now(), + deleted_by=request.user, + ) else: - initial_count = self.object.submission_count() - if delete_all_submissions: - # Update timestamp only for active records - self.object.instances.filter( - deleted_at__isnull=True).update( - deleted_at=timezone.now(), - date_modified=timezone.now(), - deleted_by=request.user) - else: - instance_ids = [ - x for x in instance_ids.split(',') if x.isdigit()] - if not instance_ids: - raise ParseError(_(u"Invalid data ids were provided.")) - - self.object.instances.filter( - id__in=instance_ids, - xform=self.object, - # do not update this timestamp when the record have - # already been deleted. - deleted_at__isnull=True - ).update( - deleted_at=timezone.now(), - date_modified=timezone.now(), - deleted_by=request.user) - - # updates the num_of_submissions for the form. - after_count = self.object.submission_count(force_update=True) - number_of_records_deleted = initial_count - after_count - - # update the date modified field of the project - self.object.project.date_modified = timezone.now() - self.object.project.save(update_fields=['date_modified']) - - # send message - send_message( - instance_id=instance_ids, target_id=self.object.id, - target_type=XFORM, user=request.user, - message_verb=SUBMISSION_DELETED) - - return Response( - data={ - "message": - "%d records were deleted" % - number_of_records_deleted - }, - status=status.HTTP_200_OK + instance_ids = [x for x in instance_ids.split(",") if x.isdigit()] + if not instance_ids: + raise ParseError(_("Invalid data ids were provided.")) + + self.object.instances.filter( + id__in=instance_ids, + xform=self.object, + # do not update this timestamp when the record have + # already been deleted. + deleted_at__isnull=True, + ).update( + deleted_at=timezone.now(), + date_modified=timezone.now(), + deleted_by=request.user, ) - elif isinstance(self.object, Instance): + # updates the num_of_submissions for the form. + after_count = self.object.submission_count(force_update=True) + number_of_records_deleted = initial_count - after_count + + # update the date modified field of the project + self.object.project.date_modified = timezone.now() + self.object.project.save(update_fields=["date_modified"]) + + # send message + send_message( + instance_id=instance_ids, + target_id=self.object.id, + target_type=XFORM, + user=request.user, + message_verb=SUBMISSION_DELETED, + ) + + return Response( + data={"message": f"{number_of_records_deleted} records were deleted"}, + status=status.HTTP_200_OK, + ) + + if isinstance(self.object, Instance): - if request.user.has_perm( - CAN_DELETE_SUBMISSION, self.object.xform): + if request.user.has_perm(CAN_DELETE_SUBMISSION, self.object.xform): instance_id = self.object.pk delete_instance(self.object, request.user) # send message send_message( - instance_id=instance_id, target_id=self.object.xform.id, - target_type=XFORM, user=request.user, - message_verb=SUBMISSION_DELETED) + instance_id=instance_id, + target_id=self.object.xform.id, + target_type=XFORM, + user=request.user, + message_verb=SUBMISSION_DELETED, + ) else: - raise PermissionDenied(_(u"You do not have delete " - u"permissions.")) + raise PermissionDenied(_("You do not have delete " "permissions.")) return Response(status=status.HTTP_204_NO_CONTENT) def retrieve(self, request, *args, **kwargs): - data_id, _format = get_data_and_form(kwargs) + """Returns API data for the targeted object.""" + _data_id, _format = get_data_and_form(kwargs) + # pylint: disable=attribute-defined-outside-init self.object = instance = self.get_object() - if _format == 'json' or _format is None or _format == 'debug': + if _format == "json" or _format is None or _format == "debug": return Response(instance.json) - elif _format == 'xml': + if _format == "xml": return Response(instance.xml) - elif _format == 'geojson': - return super(DataViewSet, self)\ - .retrieve(request, *args, **kwargs) - elif _format == Attachment.OSM: + if _format == "geojson": + return super().retrieve(request, *args, **kwargs) + if _format == Attachment.OSM: serializer = self.get_serializer(instance.osm_data.all()) return Response(serializer.data) - else: - raise ParseError( - _(u"'%(_format)s' format unknown or not implemented!" % - {'_format': _format})) - @action(methods=['GET'], detail=True) + raise ParseError(_(f"'{_format}' format unknown or not implemented!")) + + @action(methods=["GET"], detail=True) def history(self, request, *args, **kwargs): - data_id, _format = get_data_and_form(kwargs) + """ + Return submission history. + """ + _data_id, _format = get_data_and_form(kwargs) instance = self.get_object() # retrieve all history objects and return them - if _format == 'json' or _format is None or _format == 'debug': + if _format == "json" or _format is None or _format == "debug": instance_history = instance.submission_history.all() - serializer = InstanceHistorySerializer( - instance_history, many=True) + serializer = InstanceHistorySerializer(instance_history, many=True) return Response(serializer.data) - else: - raise ParseError( - _(u"'%(_format)s' format unknown or not implemented!" % - {'_format': _format})) + raise ParseError(_(f"'{_format}' format unknown or not implemented!")) + # pylint: disable=too-many-locals,too-many-branches def _set_pagination_headers( - self, xform: XForm, current_page: Union[int, str], - current_page_size: Union[int, str] = - SUBMISSION_RETRIEVAL_THRESHOLD): + self, + xform: XForm, + current_page: Union[int, str], + current_page_size: Union[int, str] = SUBMISSION_RETRIEVAL_THRESHOLD, + ): """ Sets the self.headers value for the viewset """ - import math - url = self.request.build_absolute_uri() - query = self.request.query_params.get('query') - base_url = url.split('?')[0] + query = self.request.query_params.get("query") + base_url = url.split("?")[0] if query: num_of_records = self.object_list.count() else: @@ -443,206 +481,230 @@ def _set_pagination_headers( if (current_page * current_page_size) < num_of_records: next_page_url = ( - f"{base_url}?page={current_page + 1}&" - f"page_size={current_page_size}") + f"{base_url}?page={current_page + 1}&" f"page_size={current_page_size}" + ) if current_page > 1: prev_page_url = ( - f"{base_url}?page={current_page - 1}" - f"&page_size={current_page_size}") + f"{base_url}?page={current_page - 1}" f"&page_size={current_page_size}" + ) last_page = math.ceil(num_of_records / current_page_size) - if last_page != current_page and last_page != current_page + 1: - last_page_url = ( - f"{base_url}?page={last_page}&page_size={current_page_size}" - ) + if last_page not in (current_page, current_page + 1): + last_page_url = f"{base_url}?page={last_page}&page_size={current_page_size}" if current_page != 1: - first_page_url = ( - f"{base_url}?page=1&page_size={current_page_size}" - ) + first_page_url = f"{base_url}?page=1&page_size={current_page_size}" - if not hasattr(self, 'headers'): + if not hasattr(self, "headers"): + # pylint: disable=attribute-defined-outside-init self.headers = {} for rel, link in ( - ('prev', prev_page_url), - ('next', next_page_url), - ('last', last_page_url), - ('first', first_page_url)): + ("prev", prev_page_url), + ("next", next_page_url), + ("last", last_page_url), + ("first", first_page_url), + ): if link: links.append(f'<{link}>; rel="{rel}"') - self.headers.update({'Link': ', '.join(links)}) + self.headers.update({"Link": ", ".join(links)}) + # pylint: disable=too-many-locals,too-many-branches,too-many-statements def list(self, request, *args, **kwargs): + """Returns list of data API endpoints for different forms.""" fields = request.GET.get("fields") query = request.GET.get("query", {}) sort = request.GET.get("sort") start = parse_int(request.GET.get("start")) limit = parse_int(request.GET.get("limit")) - export_type = kwargs.get('format', request.GET.get("format")) + export_type = kwargs.get("format", request.GET.get("format")) lookup_field = self.lookup_field lookup = self.kwargs.get(lookup_field) is_public_request = lookup == self.public_data_endpoint if lookup_field not in list(kwargs): + # pylint: disable=attribute-defined-outside-init self.object_list = self.filter_queryset(self.get_queryset()) serializer = self.get_serializer(self.object_list, many=True) return Response(serializer.data) if is_public_request: + # pylint: disable=attribute-defined-outside-init self.object_list = self._get_public_forms_queryset() elif lookup: - qs = self.filter_queryset( - self.get_queryset() - ).values_list('pk', 'is_merged_dataset') - xform_id, is_merged_dataset = qs[0] if qs else (lookup, False) + queryset = self.filter_queryset(self.get_queryset()).values_list( + "pk", "is_merged_dataset" + ) + xform_id, is_merged_dataset = queryset[0] if queryset else (lookup, False) pks = [xform_id] if is_merged_dataset: merged_form = MergedXForm.objects.get(pk=xform_id) - qs = merged_form.xforms.filter( - deleted_at__isnull=True).values_list( - 'id', 'num_of_submissions') + queryset = merged_form.xforms.filter( + deleted_at__isnull=True + ).values_list("id", "num_of_submissions") try: - pks, num_of_submissions = [ - list(value) for value in zip(*qs)] + pks, num_of_submissions = [list(value) for value in zip(*queryset)] num_of_submissions = sum(num_of_submissions) except ValueError: pks, num_of_submissions = [], 0 else: - num_of_submissions = XForm.objects.get( - id=xform_id).num_of_submissions + num_of_submissions = XForm.objects.get(id=xform_id).num_of_submissions + # pylint: disable=attribute-defined-outside-init self.object_list = Instance.objects.filter( - xform_id__in=pks, deleted_at=None).only('json') + xform_id__in=pks, deleted_at=None + ).only("json") # Enable ordering for XForms with Submissions that are less # than the SUBMISSION_RETRIEVAL_THRESHOLD if num_of_submissions < SUBMISSION_RETRIEVAL_THRESHOLD: - self.object_list = self.object_list.order_by('id') + # pylint: disable=attribute-defined-outside-init + self.object_list = self.object_list.order_by("id") xform = self.get_object() - self.object_list = \ - filter_queryset_xform_meta_perms(xform, request.user, - self.object_list) - tags = self.request.query_params.get('tags') - not_tagged = self.request.query_params.get('not_tagged') + # pylint: disable=attribute-defined-outside-init + self.object_list = filter_queryset_xform_meta_perms( + xform, request.user, self.object_list + ) + tags = self.request.query_params.get("tags") + not_tagged = self.request.query_params.get("not_tagged") + # pylint: disable=attribute-defined-outside-init self.object_list = filters.InstanceFilter( - self.request.query_params, - queryset=self.object_list, - request=request + self.request.query_params, queryset=self.object_list, request=request ).qs if tags and isinstance(tags, six.string_types): - tags = tags.split(',') + tags = tags.split(",") self.object_list = self.object_list.filter(tags__name__in=tags) if not_tagged and isinstance(not_tagged, six.string_types): - not_tagged = not_tagged.split(',') - self.object_list = \ - self.object_list.exclude(tags__name__in=not_tagged) + not_tagged = not_tagged.split(",") + self.object_list = self.object_list.exclude(tags__name__in=not_tagged) if ( - export_type is None or - export_type in ['json', 'jsonp', 'debug', 'xml']) \ - and hasattr(self, 'object_list'): - return self._get_data(query, fields, sort, start, limit, - is_public_request) + export_type is None or export_type in ["json", "jsonp", "debug", "xml"] + ) and hasattr(self, "object_list"): + return self._get_data(query, fields, sort, start, limit, is_public_request) xform = self.get_object() - kwargs = {'instance__xform': xform} + kwargs = {"instance__xform": xform} if export_type == Attachment.OSM: if request.GET: self.set_object_list( - query, fields, sort, start, limit, is_public_request) - kwargs = {'instance__in': self.object_list} - osm_list = OsmData.objects.filter(**kwargs).order_by('instance') + query, fields, sort, start, limit, is_public_request + ) + kwargs = {"instance__in": self.object_list} + osm_list = OsmData.objects.filter(**kwargs).order_by("instance") page = self.paginate_queryset(osm_list) serializer = self.get_serializer(page) return Response(serializer.data) - elif export_type is None or export_type in ['json']: + if export_type is None or export_type in ["json"]: # perform default viewset retrieve, no data export - return super(DataViewSet, self).list(request, *args, **kwargs) + return super().list(request, *args, **kwargs) - elif export_type == 'geojson': + if export_type == "geojson": serializer = self.get_serializer(self.object_list, many=True) return Response(serializer.data) return custom_response_handler(request, xform, query, export_type) - def set_object_list( - self, query, fields, sort, start, limit, is_public_request): + # pylint: disable=too-many-arguments + def set_object_list(self, query, fields, sort, start, limit, is_public_request): + """ + Set the submission instances queryset. + """ try: enable_etag = True if not is_public_request: xform = self.get_object() self.data_count = xform.num_of_submissions - enable_etag = self.data_count <\ - SUBMISSION_RETRIEVAL_THRESHOLD + enable_etag = self.data_count < SUBMISSION_RETRIEVAL_THRESHOLD where, where_params = get_where_clause(query) if where: - self.object_list = self.object_list.extra(where=where, - params=where_params) + # pylint: disable=attribute-defined-outside-init + self.object_list = self.object_list.extra( + where=where, params=where_params + ) if (start and limit or limit) and (not sort and not fields): start = start if start is not None else 0 limit = limit if start is None or start == 0 else start + limit + # pylint: disable=attribute-defined-outside-init self.object_list = filter_queryset_xform_meta_perms( - self.get_object(), self.request.user, self.object_list) + self.get_object(), self.request.user, self.object_list + ) + # pylint: disable=attribute-defined-outside-init self.object_list = self.object_list[start:limit] elif (sort or limit or start or fields) and not is_public_request: try: - query = \ - filter_queryset_xform_meta_perms_sql(self.get_object(), - self.request.user, - query) + query = filter_queryset_xform_meta_perms_sql( + self.get_object(), self.request.user, query + ) + # pylint: disable=attribute-defined-outside-init self.object_list = query_data( - xform, query=query, sort=sort, start_index=start, - limit=limit, fields=fields, - json_only=not self.kwargs.get('format') == 'xml') + xform, + query=query, + sort=sort, + start_index=start, + limit=limit, + fields=fields, + json_only=not self.kwargs.get("format") == "xml", + ) except NoRecordsPermission: + # pylint: disable=attribute-defined-outside-init self.object_list = [] # ETags are Disabled for XForms with Submissions that surpass # the configured SUBMISSION_RETRIEVAL_THRESHOLD setting if enable_etag: if isinstance(self.object_list, QuerySet): - self.etag_hash = get_etag_hash_from_query(self.object_list) + setattr( + self, "etag_hash", (get_etag_hash_from_query(self.object_list)) + ) else: sql, params, records = get_sql_with_params( - xform, query=query, sort=sort, start_index=start, - limit=limit, fields=fields + xform, + query=query, + sort=sort, + start_index=start, + limit=limit, + fields=fields, + ) + setattr( + self, + "etag_hash", + (get_etag_hash_from_query(records, sql, params)), ) - self.etag_hash = get_etag_hash_from_query( - records, sql, params) except ValueError as e: - raise ParseError(text(e)) + raise ParseError(str(e)) from e except DataError as e: - raise ParseError(text(e)) + raise ParseError(str(e)) from e def paginate_queryset(self, queryset): + """Returns a paginated queryset.""" if self.paginator is None: return None - return self.paginator.paginate_queryset(queryset, - self.request, - view=self, - count=self.data_count) + return self.paginator.paginate_queryset( + queryset, self.request, view=self, count=self.data_count + ) + # pylint: disable=too-many-arguments,too-many-locals def _get_data(self, query, fields, sort, start, limit, is_public_request): - self.set_object_list( - query, fields, sort, start, limit, is_public_request) + self.set_object_list(query, fields, sort, start, limit, is_public_request) - retrieval_threshold = getattr( - settings, "SUBMISSION_RETRIEVAL_THRESHOLD", 10000) - pagination_keys = [self.paginator.page_query_param, - self.paginator.page_size_query_param] + retrieval_threshold = getattr(settings, "SUBMISSION_RETRIEVAL_THRESHOLD", 10000) + pagination_keys = [ + self.paginator.page_query_param, + self.paginator.page_size_query_param, + ] query_param_keys = self.request.query_params - should_paginate = any([k in query_param_keys for k in pagination_keys]) + should_paginate = any(k in query_param_keys for k in pagination_keys) if not should_paginate and not is_public_request: # Paginate requests that try to retrieve data that surpasses @@ -653,13 +715,11 @@ def _get_data(self, query, fields, sort, start, limit, is_public_request): if should_paginate: self.paginator.page_size = retrieval_threshold - if not isinstance(self.object_list, types.GeneratorType) and \ - should_paginate: - current_page = query_param_keys.get( - self.paginator.page_query_param, 1) + if not isinstance(self.object_list, types.GeneratorType) and should_paginate: + current_page = query_param_keys.get(self.paginator.page_query_param, 1) current_page_size = query_param_keys.get( - self.paginator.page_size_query_param, - retrieval_threshold) + self.paginator.page_size_query_param, retrieval_threshold + ) self._set_pagination_headers( self.get_object(), @@ -668,12 +728,14 @@ def _get_data(self, query, fields, sort, start, limit, is_public_request): ) try: + # pylint: disable=attribute-defined-outside-init self.object_list = self.paginate_queryset(self.object_list) except OperationalError: + # pylint: disable=attribute-defined-outside-init self.object_list = self.paginate_queryset(self.object_list) - STREAM_DATA = getattr(settings, 'STREAM_DATA', False) - if STREAM_DATA: + stream_data = getattr(settings, "STREAM_DATA", False) + if stream_data: response = self._get_streaming_response() else: serializer = self.get_serializer(self.object_list, many=True) @@ -685,24 +747,27 @@ def _get_streaming_response(self): """ Get a StreamingHttpResponse response object """ + def get_json_string(item): - return json.dumps( - item.json if isinstance(item, Instance) else item) + """Returns the ``item`` Instance instance as a JSON string.""" + return json.dumps(item.json if isinstance(item, Instance) else item) - if self.kwargs.get('format') == 'xml': + if self.kwargs.get("format") == "xml": response = StreamingHttpResponse( renderers.InstanceXMLRenderer().stream_data( - self.object_list, self.get_serializer), - content_type="application/xml" + self.object_list, self.get_serializer + ), + content_type="application/xml", ) else: + # pylint: disable=http-response-with-content-type-json response = StreamingHttpResponse( json_stream(self.object_list, get_json_string), - content_type="application/json" + content_type="application/json", ) # calculate etag value and add it to response headers - if hasattr(self, 'etag_hash'): + if hasattr(self, "etag_hash"): self.set_etag_header(None, self.etag_hash) # set headers on streaming response @@ -712,5 +777,10 @@ def get_json_string(item): return response +# pylint: disable=too-many-ancestors class AuthenticatedDataViewSet(DataViewSet): + """ + Authenticated requests only. + """ + permission_classes = (ConnectViewsetPermissions,) diff --git a/onadata/apps/api/viewsets/dataview_viewset.py b/onadata/apps/api/viewsets/dataview_viewset.py index 28c6cc7a69..e9d6f9da9c 100644 --- a/onadata/apps/api/viewsets/dataview_viewset.py +++ b/onadata/apps/api/viewsets/dataview_viewset.py @@ -1,7 +1,10 @@ -from past.builtins import basestring - +# -*- coding: utf-8 -*- +""" +The /dataview API endpoint implementation. +""" from django.db.models.signals import post_delete, post_save from django.http import Http404, HttpResponseBadRequest +from django.utils.translation import gettext as _ from celery.result import AsyncResult from rest_framework import status @@ -16,8 +19,7 @@ from onadata.apps.api.tools import get_baseviewset_class from onadata.apps.logger.models.data_view import DataView from onadata.apps.viewer.models.export import Export -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.renderers import renderers @@ -25,29 +27,40 @@ from onadata.libs.serializers.dataview_serializer import DataViewSerializer from onadata.libs.serializers.xform_serializer import XFormSerializer from onadata.libs.utils import common_tags -from onadata.libs.utils.api_export_tools import (custom_response_handler, - export_async_export_response, - include_hxl_row, - process_async_export, - response_for_format) -from onadata.libs.utils.cache_tools import (PROJECT_LINKED_DATAVIEWS, - PROJ_OWNER_CACHE, - safe_delete) -from onadata.libs.utils.chart_tools import (get_chart_data_for_field, - get_field_from_field_name) +from onadata.libs.utils.api_export_tools import ( + custom_response_handler, + export_async_export_response, + include_hxl_row, + process_async_export, + response_for_format, +) +from onadata.libs.utils.cache_tools import ( + PROJECT_LINKED_DATAVIEWS, + PROJ_OWNER_CACHE, + safe_delete, +) +from onadata.libs.utils.chart_tools import ( + get_chart_data_for_field, + get_field_from_field_name, +) from onadata.libs.utils.export_tools import str_to_bool from onadata.libs.utils.model_tools import get_columns_with_hxl +# pylint: disable=invalid-name BaseViewset = get_baseviewset_class() def get_form_field_chart_url(url, field): - return u'%s?field_name=%s' % (url, field) + """ + Returns a chart's ``url`` with the field_name ``field`` parameter appended to it. + """ + return f"{url}?field_name={field}" -class DataViewViewSet(AuthenticateHeaderMixin, - CacheControlMixin, ETagsMixin, BaseViewset, - ModelViewSet): +# pylint: disable=too-many-ancestors +class DataViewViewSet( + AuthenticateHeaderMixin, CacheControlMixin, ETagsMixin, BaseViewset, ModelViewSet +): """ A simple ViewSet for viewing and editing DataViews. """ @@ -55,7 +68,7 @@ class DataViewViewSet(AuthenticateHeaderMixin, queryset = DataView.objects.select_related() serializer_class = DataViewSerializer permission_classes = [DataViewViewsetPermissions] - lookup_field = 'pk' + lookup_field = "pk" renderer_classes = api_settings.DEFAULT_RENDERER_CLASSES + [ renderers.XLSRenderer, renderers.XLSXRenderer, @@ -66,49 +79,56 @@ class DataViewViewSet(AuthenticateHeaderMixin, ] def get_serializer_class(self): - if self.action == 'data': + if self.action == "data": serializer_class = JsonDataSerializer else: serializer_class = self.serializer_class return serializer_class - @action(methods=['GET'], detail=True) - def data(self, request, format='json', **kwargs): + # pylint: disable=redefined-builtin,unused-argument + @action(methods=["GET"], detail=True) + def data(self, request, format="json", **kwargs): """Retrieve the data from the xform using this dataview""" start = request.GET.get("start") limit = request.GET.get("limit") count = request.GET.get("count") sort = request.GET.get("sort") query = request.GET.get("query") - export_type = self.kwargs.get('format', request.GET.get("format")) + export_type = self.kwargs.get("format", request.GET.get("format")) + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() - - if export_type is None or export_type in ['json', 'debug']: - data = DataView.query_data(self.object, start, limit, - str_to_bool(count), sort=sort, - filter_query=query) - if 'error' in data: - raise ParseError(data.get('error')) + if export_type is None or export_type in ["json", "debug"]: + data = DataView.query_data( + self.object, + start, + limit, + str_to_bool(count), + sort=sort, + filter_query=query, + ) + if "error" in data: + raise ParseError(data.get("error")) serializer = self.get_serializer(data, many=True) return Response(serializer.data) - else: - return custom_response_handler(request, self.object.xform, query, - export_type, - dataview=self.object) + return custom_response_handler( + request, self.object.xform, query, export_type, dataview=self.object + ) - @action(methods=['GET'], detail=True) + # pylint: disable=too-many-locals + @action(methods=["GET"], detail=True) def export_async(self, request, *args, **kwargs): + """Initiate's exports asynchronously.""" params = request.query_params - job_uuid = params.get('job_uuid') - export_type = params.get('format') - include_hxl = params.get('include_hxl', False) - include_labels = params.get('include_labels', False) - include_labels_only = params.get('include_labels_only', False) - force_xlsx = params.get('force_xlsx', False) + job_uuid = params.get("job_uuid") + export_type = params.get("format") + include_hxl = params.get("include_hxl", False) + include_labels = params.get("include_labels", False) + include_labels_only = params.get("include_labels_only", False) + force_xlsx = params.get("force_xlsx", False) query = params.get("query") dataview = self.get_object() xform = dataview.xform @@ -125,100 +145,103 @@ def export_async(self, request, *args, **kwargs): if force_xlsx is not None: force_xlsx = str_to_bool(force_xlsx) - remove_group_name = params.get('remove_group_name', False) - columns_with_hxl = get_columns_with_hxl(xform.survey.get('children')) + remove_group_name = params.get("remove_group_name", False) + columns_with_hxl = get_columns_with_hxl(xform.survey.get("children")) if columns_with_hxl and include_hxl: - include_hxl = include_hxl_row( - dataview.columns, list(columns_with_hxl) - ) + include_hxl = include_hxl_row(dataview.columns, list(columns_with_hxl)) options = { - 'remove_group_name': remove_group_name, - 'dataview_pk': dataview.pk, - 'include_hxl': include_hxl, - 'include_labels': include_labels, - 'include_labels_only': include_labels_only, - 'force_xlsx': force_xlsx, + "remove_group_name": remove_group_name, + "dataview_pk": dataview.pk, + "include_hxl": include_hxl, + "include_labels": include_labels, + "include_labels_only": include_labels_only, + "force_xlsx": force_xlsx, } if query: - options.update({'query': query}) + options.update({"query": query}) if job_uuid: job = AsyncResult(job_uuid) - if job.state == 'SUCCESS': + if job.state == "SUCCESS": export_id = job.result export = Export.objects.get(id=export_id) resp = export_async_export_response(request, export) else: - resp = { - 'job_status': job.state - } + resp = {"job_status": job.state} else: - resp = process_async_export(request, xform, export_type, - options=options) + resp = process_async_export(request, xform, export_type, options=options) - return Response(data=resp, - status=status.HTTP_202_ACCEPTED, - content_type="application/json") + return Response( + data=resp, status=status.HTTP_202_ACCEPTED, content_type="application/json" + ) - @action(methods=['GET'], detail=True) - def form(self, request, format='json', **kwargs): + # pylint: disable=redefined-builtin,unused-argument + @action(methods=["GET"], detail=True) + def form(self, request, format="json", **kwargs): + """Returns the form as either json, xml or XLS linked the dataview.""" dataview = self.get_object() xform = dataview.xform - if format not in ['json', 'xml', 'xls']: - return HttpResponseBadRequest('400 BAD REQUEST', - content_type='application/json', - status=400) + if format not in ["json", "xml", "xls"]: + return HttpResponseBadRequest( + "400 BAD REQUEST", content_type="application/json", status=400 + ) filename = xform.id_string + "." + format response = response_for_format(xform, format=format) - response['Content-Disposition'] = 'attachment; filename=' + filename + response["Content-Disposition"] = "attachment; filename=" + filename return response - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def form_details(self, request, *args, **kwargs): + """Returns the dataview's form API data.""" dataview = self.get_object() xform = dataview.xform - serializer = XFormSerializer(xform, context={'request': request}) + serializer = XFormSerializer(xform, context={"request": request}) - return Response(data=serializer.data, - content_type="application/json") + return Response(data=serializer.data, content_type="application/json") - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def charts(self, request, *args, **kwargs): + """Returns the charts data for the given dataview.""" dataview = self.get_object() xform = dataview.xform serializer = self.get_serializer(dataview) - field_name = request.query_params.get('field_name') - field_xpath = request.query_params.get('field_xpath') - fmt = kwargs.get('format', request.accepted_renderer.format) - group_by = request.query_params.get('group_by') + field_name = request.query_params.get("field_name") + field_xpath = request.query_params.get("field_xpath") + fmt = kwargs.get("format", request.accepted_renderer.format) + group_by = request.query_params.get("group_by") if field_name: field = get_field_from_field_name(field_name, xform) - field_xpath = field_name if isinstance(field, basestring) \ - else field.get_abbreviated_xpath() + field_xpath = ( + field_name if isinstance(field, str) else field.get_abbreviated_xpath() + ) - if field_xpath and field_xpath not in dataview.columns and \ - field_xpath not in [common_tags.SUBMISSION_TIME, - common_tags.SUBMITTED_BY, - common_tags.DURATION]: - raise Http404( - "Field %s does not not exist on the dataview" % field_name) + if ( + field_xpath + and field_xpath not in dataview.columns + and field_xpath + not in [ + common_tags.SUBMISSION_TIME, + common_tags.SUBMITTED_BY, + common_tags.DURATION, + ] + ): + raise Http404(_(f"Field {field_name} does not not exist on the dataview")) if field_name or field_xpath: data = get_chart_data_for_field( - field_name, xform, fmt, group_by, field_xpath, - data_view=dataview + field_name, xform, fmt, group_by, field_xpath, data_view=dataview ) - return Response(data, template_name='chart_detail.html') + return Response(data, template_name="chart_detail.html") - if fmt != 'json' and field_name is None: + if fmt != "json" and field_name is None: raise ParseError("Not supported") data = serializer.data @@ -226,54 +249,62 @@ def charts(self, request, *args, **kwargs): for field in xform.survey_elements: field_xpath = field.get_abbreviated_xpath() if field_xpath in dataview.columns: - url = reverse('dataviews-charts', kwargs={'pk': dataview.pk}, - request=request, format=fmt) + url = reverse( + "dataviews-charts", + kwargs={"pk": dataview.pk}, + request=request, + format=fmt, + ) field_url = get_form_field_chart_url(url, field.name) data["fields"][field.name] = field_url return Response(data) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def xls_export(self, request, *args, **kwargs): + """Returns the data views XLS export files.""" dataview = self.get_object() xform = dataview.xform token = None export_type = "xls" query = request.query_params.get("query", {}) - meta = request.GET.get('meta') - return custom_response_handler(request, - xform, - query, - export_type, - token, - meta, - dataview) + meta = request.GET.get("meta") + + return custom_response_handler( + request, xform, query, export_type, token, meta, dataview + ) def destroy(self, request, *args, **kwargs): + """Soft deletes the the dataview.""" dataview = self.get_object() user = request.user dataview.soft_delete(user) - safe_delete('{}{}'.format(PROJ_OWNER_CACHE, dataview.project.pk)) + safe_delete(f"{PROJ_OWNER_CACHE}{dataview.project.pk}") return Response(status=status.HTTP_204_NO_CONTENT) -def dataview_post_save_callback(sender, instance=None, created=False, - **kwargs): - safe_delete('{}{}'.format(PROJECT_LINKED_DATAVIEWS, instance.project.pk)) +# pylint: disable=unused-argument +def dataview_post_save_callback(sender, instance=None, created=False, **kwargs): + """Clear project cache post dataview save.""" + safe_delete(f"{PROJECT_LINKED_DATAVIEWS}{instance.project.pk}") def dataview_post_delete_callback(sender, instance, **kwargs): + """Clear project cache post dataview delete.""" if instance.project: - safe_delete('{}{}'.format(PROJECT_LINKED_DATAVIEWS, - instance.project.pk)) + safe_delete(f"{PROJECT_LINKED_DATAVIEWS}{instance.project.pk}") -post_save.connect(dataview_post_save_callback, - sender=DataView, - dispatch_uid='dataview_post_save_callback') +post_save.connect( + dataview_post_save_callback, + sender=DataView, + dispatch_uid="dataview_post_save_callback", +) -post_delete.connect(dataview_post_delete_callback, - sender=DataView, - dispatch_uid='dataview_post_delete_callback') +post_delete.connect( + dataview_post_delete_callback, + sender=DataView, + dispatch_uid="dataview_post_delete_callback", +) diff --git a/onadata/apps/api/viewsets/merged_xform_viewset.py b/onadata/apps/api/viewsets/merged_xform_viewset.py index af3470ae02..fb4d9bcaa0 100644 --- a/onadata/apps/api/viewsets/merged_xform_viewset.py +++ b/onadata/apps/api/viewsets/merged_xform_viewset.py @@ -16,53 +16,62 @@ from onadata.apps.logger.models import Instance, MergedXForm from onadata.libs import filters from onadata.libs.renderers import renderers -from onadata.libs.serializers.merged_xform_serializer import \ - MergedXFormSerializer +from onadata.libs.serializers.merged_xform_serializer import MergedXFormSerializer # pylint: disable=too-many-ancestors -class MergedXFormViewSet(mixins.CreateModelMixin, - mixins.DestroyModelMixin, - mixins.ListModelMixin, - mixins.RetrieveModelMixin, - viewsets.GenericViewSet): +class MergedXFormViewSet( + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + viewsets.GenericViewSet, +): """ Merged XForms viewset: create, list, retrieve, destroy """ - filter_backends = (filters.AnonDjangoObjectPermissionFilter, - filters.PublicDatasetsFilter) + filter_backends = ( + filters.AnonDjangoObjectPermissionFilter, + filters.PublicDatasetsFilter, + ) permission_classes = [XFormPermissions] - queryset = MergedXForm.objects.filter(deleted_at__isnull=True).annotate( - number_of_submissions=Sum('xforms__num_of_submissions')).all() + queryset = ( + MergedXForm.objects.filter(deleted_at__isnull=True) + .annotate(number_of_submissions=Sum("xforms__num_of_submissions")) + .all() + ) serializer_class = MergedXFormSerializer - renderer_classes = api_settings.DEFAULT_RENDERER_CLASSES + \ - [renderers.StaticXMLRenderer] + renderer_classes = api_settings.DEFAULT_RENDERER_CLASSES + [ + renderers.StaticXMLRenderer + ] # pylint: disable=unused-argument - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def form(self, *args, **kwargs): """Return XForm JSON, XLS or XML representing""" - fmt = kwargs['format'] - if fmt not in ['json', 'xml', 'xls']: + fmt = kwargs["format"] + if fmt not in ["json", "xml", "xls"]: return HttpResponseBadRequest( - '400 BAD REQUEST', content_type='application/json', status=400) + "400 BAD REQUEST", content_type="application/json", status=400 + ) merged_xform = self.get_object() data = getattr(merged_xform, fmt) - if fmt == 'json': - data = json.loads(data) + if fmt == "json": + data = json.loads(data) if isinstance(data, str) else data return Response(data) # pylint: disable=unused-argument - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def data(self, request, *args, **kwargs): """Return data from the merged xforms""" merged_xform = self.get_object() queryset = Instance.objects.filter( - xform__in=merged_xform.xforms.all()).order_by('pk') + xform__in=merged_xform.xforms.all() + ).order_by("pk") - return Response(queryset.values_list('json', flat=True)) + return Response(queryset.values_list("json", flat=True)) diff --git a/onadata/apps/api/viewsets/open_data_viewset.py b/onadata/apps/api/viewsets/open_data_viewset.py index cf47a46284..b634d50909 100644 --- a/onadata/apps/api/viewsets/open_data_viewset.py +++ b/onadata/apps/api/viewsets/open_data_viewset.py @@ -1,3 +1,7 @@ +# -*- coding=utf-8 -*- +""" +The /api/v1/open-data implementation. +""" import json import re from collections import OrderedDict @@ -7,7 +11,8 @@ from django.core.exceptions import PermissionDenied from django.http import StreamingHttpResponse from django.shortcuts import get_object_or_404 -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ + from rest_framework import status from rest_framework.decorators import action from rest_framework.response import Response @@ -20,49 +25,54 @@ from onadata.apps.logger.models.xform import XForm, question_types_to_exclude from onadata.apps.viewer.models.data_dictionary import DataDictionary from onadata.libs.data import parse_int -from onadata.libs.utils.logger_tools import remove_metadata_fields from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.pagination import StandardPageNumberPagination from onadata.libs.serializers.data_serializer import TableauDataSerializer from onadata.libs.serializers.open_data_serializer import OpenDataSerializer -from onadata.libs.utils.common_tools import json_stream from onadata.libs.utils.common_tags import ( ATTACHMENTS, - NOTES, GEOLOCATION, MULTIPLE_SELECT_TYPE, + NA_REP, + NOTES, REPEAT_SELECT_TYPE, - NA_REP) +) +from onadata.libs.utils.common_tools import json_stream +from onadata.libs.utils.logger_tools import remove_metadata_fields BaseViewset = get_baseviewset_class() -IGNORED_FIELD_TYPES = ['select one', 'select multiple'] +IGNORED_FIELD_TYPES = ["select one", "select multiple"] # index tags -DEFAULT_OPEN_TAG = '[' -DEFAULT_CLOSE_TAG = ']' +DEFAULT_OPEN_TAG = "[" +DEFAULT_CLOSE_TAG = "]" DEFAULT_INDEX_TAGS = (DEFAULT_OPEN_TAG, DEFAULT_CLOSE_TAG) -DEFAULT_NA_REP = getattr(settings, 'NA_REP', NA_REP) +DEFAULT_NA_REP = getattr(settings, "NA_REP", NA_REP) +# pylint: disable=invalid-name def replace_special_characters_with_underscores(data): + """Replaces special characters with underscores.""" return [re.sub(r"\W", r"_", a) for a in data] -def process_tableau_data(data, xform): +# pylint: disable=too-many-locals,too-many-statements +def process_tableau_data(data, xform): # noqa C901 """ Streamlines the row header fields with the column header fields for the same form. - Handles Flattenning repeat data for tableau + Handles Flattening repeat data for tableau """ - def get_xpath(key, nested_key): - val = nested_key.split('/') - nested_key_diff = val[len(key.split('/')):] - xpaths = key + f'[{index}]/' + '/'.join(nested_key_diff) + def get_xpath(key, nested_key, index): + val = nested_key.split("/") + start_index = key.split("/").__len__() + nested_key_diff = val[start_index:] + xpaths = key + f"[{index}]/" + "/".join(nested_key_diff) return xpaths - def get_updated_data_dict(key, value, data_dict): + def get_updated_data_dict(key, value, data_dict, index=0): """ Generates key, value pairs for select multiple question types. Defining the new xpaths from the @@ -72,13 +82,13 @@ def get_updated_data_dict(key, value, data_dict): if isinstance(value, str) and data_dict: choices = value.split(" ") for choice in choices: - xpaths = f'{key}/{choice}' + xpaths = f"{key}/{choice}" data_dict[xpaths] = choice elif isinstance(value, list): try: for item in value: for (nested_key, nested_val) in item.items(): - xpath = get_xpath(key, nested_key) + xpath = get_xpath(key, nested_key, index) data_dict[xpath] = nested_val except AttributeError: data_dict[key] = value @@ -104,18 +114,17 @@ def get_ordered_repeat_value(key, item, index): # generate ["children", index, "immunization/polio_1"] for (nested_key, nested_val) in item_list.items(): qstn_type = xform.get_element(nested_key).type - xpaths = get_xpath(key, nested_key) + xpaths = get_xpath(key, nested_key, index) if qstn_type == MULTIPLE_SELECT_TYPE: - data = get_updated_data_dict( - xpaths, nested_val, data) + data = get_updated_data_dict(xpaths, nested_val, data, index) elif qstn_type == REPEAT_SELECT_TYPE: - data = get_updated_data_dict( - xpaths, nested_val, data) + data = get_updated_data_dict(xpaths, nested_val, data, index) else: data[xpaths] = nested_val return data result = [] + # pylint: disable=too-many-nested-blocks if data: headers = xform.get_headers() tableau_headers = remove_metadata_fields(headers) @@ -124,7 +133,10 @@ def get_ordered_repeat_value(key, item, index): flat_dict = dict.fromkeys(diff, None) for (key, value) in row.items(): if isinstance(value, list) and key not in [ - ATTACHMENTS, NOTES, GEOLOCATION]: + ATTACHMENTS, + NOTES, + GEOLOCATION, + ]: for index, item in enumerate(value, start=1): # order repeat according to xform order item = get_ordered_repeat_value(key, item, index) @@ -133,15 +145,13 @@ def get_ordered_repeat_value(key, item, index): try: qstn_type = xform.get_element(key).type if qstn_type == MULTIPLE_SELECT_TYPE: - flat_dict = get_updated_data_dict( - key, value, flat_dict) - if qstn_type == 'geopoint': - parts = value.split(' ') - gps_xpaths = \ - DataDictionary.get_additional_geopoint_xpaths( - key) - gps_parts = dict( - [(xpath, None) for xpath in gps_xpaths]) + flat_dict = get_updated_data_dict(key, value, flat_dict) + if qstn_type == "geopoint": + parts = value.split(" ") + gps_xpaths = DataDictionary.get_additional_geopoint_xpaths( + key + ) + gps_parts = {xpath: None for xpath in gps_xpaths} if len(parts) == 4: gps_parts = dict(zip(gps_xpaths, parts)) flat_dict.update(gps_parts) @@ -154,58 +164,59 @@ def get_ordered_repeat_value(key, item, index): return result -class OpenDataViewSet(ETagsMixin, CacheControlMixin, - BaseViewset, ModelViewSet): - permission_classes = (OpenDataViewSetPermissions, ) +# pylint: disable=too-many-ancestors +class OpenDataViewSet(ETagsMixin, CacheControlMixin, BaseViewset, ModelViewSet): + """The /api/v1/open-data API endpoint.""" + + permission_classes = (OpenDataViewSetPermissions,) queryset = OpenData.objects.filter() - lookup_field = 'uuid' + lookup_field = "uuid" serializer_class = OpenDataSerializer flattened_dict = {} MAX_INSTANCES_PER_REQUEST = 1000 pagination_class = StandardPageNumberPagination + # pylint: disable=no-self-use def get_tableau_type(self, xform_type): - ''' + """ Returns a tableau-supported type based on a xform type. - ''' + """ tableau_types = { - 'integer': 'int', - 'decimal': 'float', - 'dateTime': 'datetime', - 'text': 'string' + "integer": "int", + "decimal": "float", + "dateTime": "datetime", + "text": "string", } - return tableau_types.get(xform_type, 'string') + return tableau_types.get(xform_type, "string") def flatten_xform_columns(self, json_of_columns_fields): - ''' + """ Flattens a json of column fields and the result is set to a class variable. - ''' + """ for a in json_of_columns_fields: - self.flattened_dict[a.get('name')] = self.get_tableau_type( - a.get('type')) + self.flattened_dict[a.get("name")] = self.get_tableau_type(a.get("type")) # using IGNORED_FIELD_TYPES so that choice values are not included. - if a.get('children') and a.get('type') not in IGNORED_FIELD_TYPES: - self.flatten_xform_columns(a.get('children')) + if a.get("children") and a.get("type") not in IGNORED_FIELD_TYPES: + self.flatten_xform_columns(a.get("children")) def get_tableau_column_headers(self): - ''' + """ Retrieve columns headers that are valid in tableau. - ''' + """ tableau_colulmn_headers = [] - def append_to_tableau_colulmn_headers(header, question_type=None): - quest_type = 'string' + def _append_to_tableau_colulmn_headers(header, question_type=None): + quest_type = "string" if question_type: quest_type = question_type # alias can be updated in the future to question labels - tableau_colulmn_headers.append({ - 'id': header, - 'dataType': quest_type, - 'alias': header - }) + tableau_colulmn_headers.append( + {"id": header, "dataType": quest_type, "alias": header} + ) + # Remove metadata fields from the column headers # Calling set to remove duplicates in group data xform_headers = set(remove_metadata_fields(self.xform_headers)) @@ -214,33 +225,34 @@ def append_to_tableau_colulmn_headers(header, question_type=None): # tableau. for header in xform_headers: for quest_name, quest_type in self.flattened_dict.items(): - if header == quest_name or header.endswith('_%s' % quest_name): - append_to_tableau_colulmn_headers(header, quest_type) + if header == quest_name or header.endswith(f"_{quest_name}"): + _append_to_tableau_colulmn_headers(header, quest_type) break else: - if header == '_id': - append_to_tableau_colulmn_headers(header, "int") + if header == "_id": + _append_to_tableau_colulmn_headers(header, "int") else: - append_to_tableau_colulmn_headers(header) + _append_to_tableau_colulmn_headers(header) return tableau_colulmn_headers - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def data(self, request, **kwargs): """ Streams submission data response matching uuid in the request. """ + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() # get greater than value and cast it to an int - gt_id = request.query_params.get('gt_id') + gt_id = request.query_params.get("gt_id") gt_id = gt_id and parse_int(gt_id) - count = request.query_params.get('count') + count = request.query_params.get("count") pagination_keys = [ self.paginator.page_query_param, - self.paginator.page_size_query_param + self.paginator.page_size_query_param, ] query_param_keys = request.query_params - should_paginate = any([k in query_param_keys for k in pagination_keys]) + should_paginate = any(k in query_param_keys for k in pagination_keys) data = [] if isinstance(self.object.content_object, XForm): @@ -249,40 +261,45 @@ def data(self, request, **kwargs): xform = self.object.content_object if xform.is_merged_dataset: - qs_kwargs = {'xform_id__in': list( - xform.mergedxform.xforms.values_list('pk', flat=True))} + qs_kwargs = { + "xform_id__in": list( + xform.mergedxform.xforms.values_list("pk", flat=True) + ) + } else: - qs_kwargs = {'xform_id': xform.pk} + qs_kwargs = {"xform_id": xform.pk} if gt_id: - qs_kwargs.update({'id__gt': gt_id}) + qs_kwargs.update({"id__gt": gt_id}) # Filter out deleted submissions instances = Instance.objects.filter( - **qs_kwargs, deleted_at__isnull=True).order_by('pk') + **qs_kwargs, deleted_at__isnull=True + ).order_by("pk") if count: - return Response({'count': instances.count()}) + return Response({"count": instances.count()}) if should_paginate: instances = self.paginate_queryset(instances) data = process_tableau_data( - TableauDataSerializer(instances, many=True).data, xform) + TableauDataSerializer(instances, many=True).data, xform + ) return self.get_streaming_response(data) return Response(data) + # pylint: disable=no-self-use def get_streaming_response(self, data): """Get a StreamingHttpResponse response object""" def get_json_string(item): - return json.dumps({ - re.sub(r"\W", r"_", a): b for a, b in item.items()}) + return json.dumps({re.sub(r"\W", r"_", a): b for a, b in item.items()}) + # pylint: disable=http-response-with-content-type-json response = StreamingHttpResponse( - json_stream(data, get_json_string), - content_type="application/json" + json_stream(data, get_json_string), content_type="application/json" ) # set headers on streaming response @@ -292,57 +309,64 @@ def get_json_string(item): return response def destroy(self, request, *args, **kwargs): + """Deletes an OpenData object.""" self.get_object().delete() return Response(status=status.HTTP_204_NO_CONTENT) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def schema(self, request, **kwargs): + """Tableau schema - headers and table alias.""" + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() if isinstance(self.object.content_object, XForm): xform = self.object.content_object headers = xform.get_headers() - self.xform_headers = replace_special_characters_with_underscores( - headers) + self.xform_headers = replace_special_characters_with_underscores(headers) - xform_json = json.loads(xform.json) + xform_json = xform.json_dict() self.flatten_xform_columns( - json_of_columns_fields=xform_json.get('children')) + json_of_columns_fields=xform_json.get("children") + ) tableau_column_headers = self.get_tableau_column_headers() data = { - 'column_headers': tableau_column_headers, - 'connection_name': "%s_%s" % (xform.project_id, - xform.id_string), - 'table_alias': xform.title + "column_headers": tableau_column_headers, + "connection_name": f"{xform.project_id}_{xform.id_string}", + "table_alias": xform.title, } return Response(data=data, status=status.HTTP_200_OK) return Response(status=status.HTTP_404_NOT_FOUND) - @action(methods=['GET'], detail=False) + # pylint: disable=no-self-use + @action(methods=["GET"], detail=False) def uuid(self, request, *args, **kwargs): - data_type = request.query_params.get('data_type') - object_id = request.query_params.get('object_id') + """Respond with the OpenData uuid.""" + data_type = request.query_params.get("data_type") + object_id = request.query_params.get("object_id") if not data_type or not object_id: return Response( data="Query params data_type and object_id are required", - status=status.HTTP_400_BAD_REQUEST) + status=status.HTTP_400_BAD_REQUEST, + ) - if data_type == 'xform': + if data_type == "xform": xform = get_object_or_404(XForm, id=object_id) if request.user.has_perm("change_xform", xform): - ct = ContentType.objects.get_for_model(xform) + content_type = ContentType.objects.get_for_model(xform) _open_data = get_object_or_404( - OpenData, object_id=object_id, content_type=ct) + OpenData, object_id=object_id, content_type=content_type + ) if _open_data: return Response( - data={'uuid': _open_data.uuid}, - status=status.HTTP_200_OK) + data={"uuid": _open_data.uuid}, status=status.HTTP_200_OK + ) else: raise PermissionDenied( - _((u"You do not have permission to perform this action."))) + _(("You do not have permission to perform this action.")) + ) return Response(status=status.HTTP_404_NOT_FOUND) diff --git a/onadata/apps/api/viewsets/osm_viewset.py b/onadata/apps/api/viewsets/osm_viewset.py index 55efe858c0..243d75ae86 100644 --- a/onadata/apps/api/viewsets/osm_viewset.py +++ b/onadata/apps/api/viewsets/osm_viewset.py @@ -1,145 +1,157 @@ +# -*- coding: utf-8 -*- +""" +The osm API endpoint. +""" from django.http import HttpResponsePermanentRedirect from django.shortcuts import get_object_or_404 -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ -from rest_framework.viewsets import ReadOnlyModelViewSet from rest_framework.exceptions import ParseError from rest_framework.permissions import AllowAny from rest_framework.renderers import JSONRenderer from rest_framework.response import Response from rest_framework.reverse import reverse +from rest_framework.viewsets import ReadOnlyModelViewSet +from onadata.apps.api.tools import get_baseviewset_class from onadata.apps.logger.models import OsmData -from onadata.apps.logger.models.xform import XForm from onadata.apps.logger.models.attachment import Attachment from onadata.apps.logger.models.instance import Instance -from onadata.libs.renderers import renderers -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.apps.logger.models.xform import XForm +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin -from onadata.libs.serializers.data_serializer import OSMSerializer -from onadata.libs.serializers.data_serializer import OSMSiteMapSerializer -from onadata.apps.api.tools import get_baseviewset_class +from onadata.libs.renderers import renderers +from onadata.libs.serializers.data_serializer import OSMSerializer, OSMSiteMapSerializer +# pylint: disable=invalid-name BaseViewset = get_baseviewset_class() -SAFE_METHODS = ['GET', 'HEAD', 'OPTIONS'] +SAFE_METHODS = ["GET", "HEAD", "OPTIONS"] -class OsmViewSet(AuthenticateHeaderMixin, - CacheControlMixin, ETagsMixin, BaseViewset, - ReadOnlyModelViewSet): +# pylint: disable=too-many-ancestors +class OsmViewSet( + AuthenticateHeaderMixin, + CacheControlMixin, + ETagsMixin, + BaseViewset, + ReadOnlyModelViewSet, +): """ -This endpoint provides public access to OSM submitted data in OSM format. -No authentication is required. Where: + This endpoint provides public access to OSM submitted data in OSM format. + No authentication is required. Where: -* `pk` - the form unique identifier -* `dataid` - submission data unique identifier -* `owner` - username of the owner(user/organization) of the data point + * `pk` - the form unique identifier + * `dataid` - submission data unique identifier + * `owner` - username of the owner(user/organization) of the data point -## GET JSON List of data end points + ## GET JSON List of data end points -Lists the data endpoints accessible to requesting user, for anonymous access -a list of public data endpoints is returned. + Lists the data endpoints accessible to requesting user, for anonymous access + a list of public data endpoints is returned. -
-GET /api/v1/osm
-
+ 
+    GET /api/v1/osm
+
-> Example -> -> curl -X GET https://ona.io/api/v1/osm + > Example + > + > curl -X GET https://ona.io/api/v1/osm -## OSM + ## OSM -The `.osm` file format concatenates all the files for a form or individual - submission. When the `.json` endpoint is accessed, the individual osm files - are listed on the `_attachments` key. + The `.osm` file format concatenates all the files for a form or individual + submission. When the `.json` endpoint is accessed, the individual osm files + are listed on the `_attachments` key. -### OSM endpoint for all osm files uploaded to a form concatenated. + ### OSM endpoint for all osm files uploaded to a form concatenated. -
-GET /api/v1/osm/{pk}.osm
-
+ 
+    GET /api/v1/osm/{pk}.osm
+
-> Example -> -> curl -X GET https://ona.io/api/v1/osm/28058.osm + > Example + > + > curl -X GET https://ona.io/api/v1/osm/28058.osm -### OSM endpoint with all osm files for a specific submission concatenated. + ### OSM endpoint with all osm files for a specific submission concatenated. -
-GET /api/v1/osm/{pk}/{data_id}.osm
-
+ 
+    GET /api/v1/osm/{pk}/{data_id}.osm
+
-> Example -> -> curl -X GET https://ona.io/api/v1/osm/28058/20.osm + > Example + > + > curl -X GET https://ona.io/api/v1/osm/28058/20.osm + """ -""" renderer_classes = [ renderers.OSMRenderer, JSONRenderer, ] serializer_class = OSMSerializer - permission_classes = (AllowAny, ) - lookup_field = 'pk' - lookup_fields = ('pk', 'dataid') + permission_classes = (AllowAny,) + lookup_field = "pk" + lookup_fields = ("pk", "dataid") extra_lookup_fields = None - public_data_endpoint = 'public' + public_data_endpoint = "public" queryset = XForm.objects.filter().select_related() def get_serializer_class(self): - pk = self.kwargs.get('pk') - if self.action == 'list' and pk is None: + """Returns the OSMSiteMapSerializer class when list API is invoked.""" + form_pk = self.kwargs.get("pk") + if self.action == "list" and form_pk is None: return OSMSiteMapSerializer - return super(OsmViewSet, self).get_serializer_class() + return super().get_serializer_class() def filter_queryset(self, queryset): - pk = self.kwargs.get('pk') - if pk: - queryset = queryset.filter(pk=pk) + """Filters the queryset using the ``pk`` when used.""" + form_pk = self.kwargs.get("pk") + if form_pk: + queryset = queryset.filter(pk=form_pk) - return super(OsmViewSet, self).filter_queryset(queryset) + return super().filter_queryset(queryset) def get_object(self): - obj = super(OsmViewSet, self).get_object() + """Returns the Instance object using the ``pk`` and ``dataid`` lookup values.""" + obj = super().get_object() pk_lookup, dataid_lookup = self.lookup_fields - pk = self.kwargs.get(pk_lookup) + form_pk = self.kwargs.get(pk_lookup) dataid = self.kwargs.get(dataid_lookup) - if pk is not None and dataid is not None: + if form_pk is not None and dataid is not None: try: int(dataid) - except ValueError: - raise ParseError(_(u"Invalid dataid %(dataid)s" - % {'dataid': dataid})) + except ValueError as e: + raise ParseError(_(f"Invalid dataid {dataid}")) from e - obj = get_object_or_404(Instance, pk=dataid, xform__pk=pk) + obj = get_object_or_404(Instance, pk=dataid, xform__pk=form_pk) return obj def retrieve(self, request, *args, **kwargs): - fmt = kwargs.get('format', request.accepted_renderer.format) - if fmt != 'osm': + """Returns a single Instance JSON object API response""" + fmt = kwargs.get("format", request.accepted_renderer.format) + if fmt != "osm": pk_lookup, dataid_lookup = self.lookup_fields - pk = self.kwargs.get(pk_lookup) + form_pk = self.kwargs.get(pk_lookup) dataid = self.kwargs.get(dataid_lookup) - kwargs = {'pk': pk, 'format': 'osm'} - viewname = 'osm-list' + kwargs = {"pk": form_pk, "format": "osm"} + viewname = "osm-list" if dataid: kwargs[dataid_lookup] = dataid - viewname = 'osm-detail' + viewname = "osm-detail" return HttpResponsePermanentRedirect( - reverse(viewname, kwargs=kwargs, request=request)) + reverse(viewname, kwargs=kwargs, request=request) + ) instance = self.get_object() if isinstance(instance, XForm): @@ -151,13 +163,18 @@ def retrieve(self, request, *args, **kwargs): return Response(serializer.data) def list(self, request, *args, **kwargs): - fmt = kwargs.get('format', request.accepted_renderer.format) - pk = kwargs.get('pk') - if pk: - if fmt != 'osm': + """Returns a list of URLs to the individual XForm OSM data.""" + fmt = kwargs.get("format", request.accepted_renderer.format) + form_pk = kwargs.get("pk") + if form_pk: + if fmt != "osm": return HttpResponsePermanentRedirect( - reverse('osm-list', kwargs={'pk': pk, 'format': 'osm'}, - request=request)) + reverse( + "osm-list", + kwargs={"pk": form_pk, "format": "osm"}, + request=request, + ) + ) instance = self.filter_queryset(self.get_queryset()) osm_list = OsmData.objects.filter(instance__xform__in=instance) page = self.paginate_queryset(osm_list) @@ -168,15 +185,21 @@ def list(self, request, *args, **kwargs): return Response(serializer.data) - if fmt == 'osm': + if fmt == "osm": return HttpResponsePermanentRedirect( - reverse('osm-list', kwargs={'format': 'json'}, - request=request)) - instances = Attachment.objects.filter(extension='osm').values( - 'instance__xform', 'instance__xform__user__username', - 'instance__xform__title', 'instance__xform__id_string')\ - .order_by('instance__xform__id')\ - .distinct('instance__xform__id') + reverse("osm-list", kwargs={"format": "json"}, request=request) + ) + instances = ( + Attachment.objects.filter(extension="osm") + .values( + "instance__xform", + "instance__xform__user__username", + "instance__xform__title", + "instance__xform__id_string", + ) + .order_by("instance__xform__id") + .distinct("instance__xform__id") + ) serializer = self.get_serializer(instances, many=True) - return Response(serializer.data, content_type='application/json') + return Response(serializer.data, content_type="application/json") diff --git a/onadata/apps/api/viewsets/project_viewset.py b/onadata/apps/api/viewsets/project_viewset.py index c21b07eecc..3c0da0d1c0 100644 --- a/onadata/apps/api/viewsets/project_viewset.py +++ b/onadata/apps/api/viewsets/project_viewset.py @@ -1,8 +1,10 @@ -from distutils.util import strtobool - +# -*- coding=utf-8 -*- +""" +The /projects API endpoint implementation. +""" +from django.core.cache import cache from django.core.mail import send_mail from django.shortcuts import get_object_or_404 -from django.core.cache import cache from rest_framework import status from rest_framework.decorators import action @@ -15,124 +17,132 @@ from onadata.apps.logger.models import Project, XForm from onadata.apps.main.models import UserProfile from onadata.apps.main.models.meta_data import MetaData -from onadata.libs.filters import (AnonUserProjectFilter, ProjectOwnerFilter, - TagFilter) -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.data import strtobool +from onadata.libs.filters import AnonUserProjectFilter, ProjectOwnerFilter, TagFilter +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.mixins.labels_mixin import LabelsMixin from onadata.libs.mixins.profiler_mixin import ProfilerMixin -from onadata.libs.serializers.project_serializer import \ - (BaseProjectSerializer, ProjectSerializer) -from onadata.libs.serializers.share_project_serializer import \ - (RemoveUserFromProjectSerializer, ShareProjectSerializer) -from onadata.libs.serializers.user_profile_serializer import \ - UserProfileSerializer -from onadata.libs.utils.cache_tools import ( - PROJ_OWNER_CACHE, safe_delete) -from onadata.libs.serializers.xform_serializer import (XFormCreateSerializer, - XFormSerializer) +from onadata.libs.serializers.project_serializer import ( + BaseProjectSerializer, + ProjectSerializer, +) +from onadata.libs.serializers.share_project_serializer import ( + RemoveUserFromProjectSerializer, + ShareProjectSerializer, +) +from onadata.libs.serializers.user_profile_serializer import UserProfileSerializer +from onadata.libs.serializers.xform_serializer import ( + XFormCreateSerializer, + XFormSerializer, +) +from onadata.libs.utils.cache_tools import PROJ_OWNER_CACHE, safe_delete from onadata.libs.utils.common_tools import merge_dicts from onadata.libs.utils.export_tools import str_to_bool from onadata.settings.common import DEFAULT_FROM_EMAIL, SHARE_PROJECT_SUBJECT +# pylint: disable=invalid-name BaseViewset = get_baseviewset_class() -class ProjectViewSet(AuthenticateHeaderMixin, - CacheControlMixin, - ETagsMixin, LabelsMixin, ProfilerMixin, - BaseViewset, ModelViewSet): +# pylint: disable=too-many-ancestors +class ProjectViewSet( + AuthenticateHeaderMixin, + CacheControlMixin, + ETagsMixin, + LabelsMixin, + ProfilerMixin, + BaseViewset, + ModelViewSet, +): """ List, Retrieve, Update, Create Project and Project Forms. """ + + # pylint: disable=no-member queryset = Project.objects.filter(deleted_at__isnull=True).select_related() serializer_class = ProjectSerializer - lookup_field = 'pk' + lookup_field = "pk" extra_lookup_fields = None permission_classes = [ProjectPermissions] - filter_backends = (AnonUserProjectFilter, - ProjectOwnerFilter, - TagFilter) + filter_backends = (AnonUserProjectFilter, ProjectOwnerFilter, TagFilter) def get_serializer_class(self): - action = self.action - - if action == "list": - serializer_class = BaseProjectSerializer - else: - serializer_class = \ - super(ProjectViewSet, self).get_serializer_class() - - return serializer_class + """Return BaseProjectSerializer class when listing projects.""" + if self.action == "list": + return BaseProjectSerializer + return super().get_serializer_class() def get_queryset(self): - if self.request.method.upper() in ['GET', 'OPTIONS']: + """Use 'prepared' prefetched queryset for GET requests.""" + if self.request.method.upper() in ["GET", "OPTIONS"]: self.queryset = Project.prefetched.filter( - deleted_at__isnull=True, organization__is_active=True) + deleted_at__isnull=True, organization__is_active=True + ) - return super(ProjectViewSet, self).get_queryset() + return super().get_queryset() def update(self, request, *args, **kwargs): - project_id = kwargs.get('pk') - response = super(ProjectViewSet, self).update(request, *args, **kwargs) - cache.set(f'{PROJ_OWNER_CACHE}{project_id}', response.data) + """Updates project properties and set's cache with the updated records.""" + project_id = kwargs.get("pk") + response = super().update(request, *args, **kwargs) + cache.set(f"{PROJ_OWNER_CACHE}{project_id}", response.data) return response def retrieve(self, request, *args, **kwargs): - """ Retrieve single project """ - project_id = kwargs.get('pk') - project = cache.get(f'{PROJ_OWNER_CACHE}{project_id}') + """Retrieve single project""" + project_id = kwargs.get("pk") + project = cache.get(f"{PROJ_OWNER_CACHE}{project_id}") if project: return Response(project) + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() - serializer = ProjectSerializer( - self.object, context={'request': request}) + serializer = ProjectSerializer(self.object, context={"request": request}) return Response(serializer.data) - @action(methods=['POST', 'GET'], detail=True) + @action(methods=["POST", "GET"], detail=True) def forms(self, request, **kwargs): """Add a form to a project or list forms for the project. The request key `xls_file` holds the XLSForm file object. """ + # pylint: disable=attribute-defined-outside-init project = self.object = self.get_object() - if request.method.upper() == 'POST': + if request.method.upper() == "POST": survey = utils.publish_project_xform(request, project) if isinstance(survey, XForm): - if 'formid' in request.data: + if "formid" in request.data: serializer_cls = XFormSerializer else: serializer_cls = XFormCreateSerializer - serializer = serializer_cls(survey, - context={'request': request}) + serializer = serializer_cls(survey, context={"request": request}) - published_by_formbuilder = request.data.get( - 'published_by_formbuilder' - ) + published_by_formbuilder = request.data.get("published_by_formbuilder") if str_to_bool(published_by_formbuilder): - MetaData.published_by_formbuilder(survey, 'True') + MetaData.published_by_formbuilder(survey, "True") - return Response(serializer.data, - status=status.HTTP_201_CREATED) + return Response(serializer.data, status=status.HTTP_201_CREATED) return Response(survey, status=status.HTTP_400_BAD_REQUEST) xforms = XForm.objects.filter(project=project) - serializer = XFormSerializer(xforms, context={'request': request}, - many=True) + serializer = XFormSerializer(xforms, context={"request": request}, many=True) return Response(serializer.data) - @action(methods=['PUT'], detail=True) + @action(methods=["PUT"], detail=True) def share(self, request, *args, **kwargs): + """ + Allow sharing of a project to a user. + """ + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() - data = merge_dicts(request.data.dict(), {'project': self.object.pk}) + data = merge_dicts(request.data.dict(), {"project": self.object.pk}) remove = data.get("remove") if remove and remove is not isinstance(remove, bool): @@ -144,7 +154,7 @@ def share(self, request, *args, **kwargs): serializer = ShareProjectSerializer(data=data) if serializer.is_valid(): serializer.save() - email_msg = data.get('email_msg') + email_msg = data.get("email_msg") if email_msg: # send out email message. try: @@ -154,43 +164,53 @@ def share(self, request, *args, **kwargs): user = instance.user send_mail( SHARE_PROJECT_SUBJECT.format(self.object.name), - email_msg, DEFAULT_FROM_EMAIL, (user.email, )) + email_msg, + DEFAULT_FROM_EMAIL, + (user.email,), + ) else: - send_mail(SHARE_PROJECT_SUBJECT.format(self.object.name), - email_msg, DEFAULT_FROM_EMAIL, (user.email,)) + send_mail( + SHARE_PROJECT_SUBJECT.format(self.object.name), + email_msg, + DEFAULT_FROM_EMAIL, + (user.email,), + ) else: - return Response(data=serializer.errors, - status=status.HTTP_400_BAD_REQUEST) + return Response(data=serializer.errors, status=status.HTTP_400_BAD_REQUEST) # clear cache - safe_delete(f'{PROJ_OWNER_CACHE}{self.object.pk}') + safe_delete(f"{PROJ_OWNER_CACHE}{self.object.pk}") return Response(status=status.HTTP_204_NO_CONTENT) - @action(methods=['DELETE', 'GET', 'POST'], detail=True) + @action(methods=["DELETE", "GET", "POST"], detail=True) def star(self, request, *args, **kwargs): + """ + Allows to add a user that stars a project. + """ user = request.user - self.object = project = get_object_or_404(Project, - pk=kwargs.get('pk')) + # pylint: disable=attribute-defined-outside-init + self.object = project = get_object_or_404(Project, pk=kwargs.get("pk")) - if request.method == 'DELETE': + if request.method == "DELETE": project.user_stars.remove(user) project.save() - elif request.method == 'POST': + elif request.method == "POST": project.user_stars.add(user) project.save() - elif request.method == 'GET': - users = project.user_stars.values('pk') + elif request.method == "GET": + users = project.user_stars.values("pk") user_profiles = UserProfile.objects.filter(user__in=users) - serializer = UserProfileSerializer(user_profiles, - context={'request': request}, - many=True) + serializer = UserProfileSerializer( + user_profiles, context={"request": request}, many=True + ) return Response(serializer.data) return Response(status=status.HTTP_204_NO_CONTENT) def destroy(self, request, *args, **kwargs): + """ "Soft deletes a project""" project = self.get_object() user = request.user project.soft_delete(user) diff --git a/onadata/apps/api/viewsets/team_viewset.py b/onadata/apps/api/viewsets/team_viewset.py index 4a9bae83d7..2127746cb6 100644 --- a/onadata/apps/api/viewsets/team_viewset.py +++ b/onadata/apps/api/viewsets/team_viewset.py @@ -1,83 +1,99 @@ -from distutils.util import strtobool - -from django.contrib.auth.models import User -from django.utils.translation import ugettext as _ +# -*- coding=utf-8 -*- +""" +The /teams API endpoint implementation. +""" +from django.contrib.auth import get_user_model +from django.utils.translation import gettext as _ from rest_framework import status -from rest_framework_guardian.filters import ObjectPermissionsFilter from rest_framework.decorators import action from rest_framework.permissions import DjangoObjectPermissions from rest_framework.response import Response from rest_framework.viewsets import ModelViewSet +from rest_framework_guardian.filters import ObjectPermissionsFilter from onadata.apps.api.models import Team -from onadata.apps.api.tools import (add_user_to_team, get_baseviewset_class, - remove_user_from_team) +from onadata.apps.api.tools import ( + add_user_to_team, + get_baseviewset_class, + remove_user_from_team, +) +from onadata.libs.data import strtobool from onadata.libs.filters import TeamOrgFilter -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin -from onadata.libs.serializers.share_team_project_serializer import \ - (RemoveTeamFromProjectSerializer, ShareTeamProjectSerializer) +from onadata.libs.serializers.share_team_project_serializer import ( + RemoveTeamFromProjectSerializer, + ShareTeamProjectSerializer, +) from onadata.libs.serializers.team_serializer import TeamSerializer from onadata.libs.utils.common_tools import merge_dicts +# pylint: disable=invalid-name BaseViewset = get_baseviewset_class() +# pylint: disable=invalid-name +User = get_user_model() -class TeamViewSet(AuthenticateHeaderMixin, - CacheControlMixin, ETagsMixin, - BaseViewset, - ModelViewSet): +# pylint: disable=too-many-ancestors +class TeamViewSet( + AuthenticateHeaderMixin, CacheControlMixin, ETagsMixin, BaseViewset, ModelViewSet +): """ This endpoint allows you to create, update and view team information. """ + queryset = Team.objects.all() serializer_class = TeamSerializer - lookup_field = 'pk' + lookup_field = "pk" extra_lookup_fields = None permission_classes = [DjangoObjectPermissions] - filter_backends = (ObjectPermissionsFilter, - TeamOrgFilter) + filter_backends = (ObjectPermissionsFilter, TeamOrgFilter) - @action(methods=['DELETE', 'GET', 'POST'], detail=True) + @action(methods=["DELETE", "GET", "POST"], detail=True) def members(self, request, *args, **kwargs): + """ + Returns members of an organization. + """ team = self.get_object() data = {} status_code = status.HTTP_200_OK - if request.method in ['DELETE', 'POST']: - username = request.data.get('username') or\ - request.query_params.get('username') + if request.method in ["DELETE", "POST"]: + username = request.data.get("username") or request.query_params.get( + "username" + ) if username: try: user = User.objects.get(username__iexact=username) except User.DoesNotExist: status_code = status.HTTP_400_BAD_REQUEST - data['username'] = [ - _(u"User `%(username)s` does not exist." - % {'username': username})] + data["username"] = [_(f"User `{username}` does not exist.")] else: - if request.method == 'POST': + if request.method == "POST": add_user_to_team(team, user) - elif request.method == 'DELETE': + elif request.method == "DELETE": remove_user_from_team(team, user) status_code = status.HTTP_201_CREATED else: status_code = status.HTTP_400_BAD_REQUEST - data['username'] = [_(u"This field is required.")] + data["username"] = [_("This field is required.")] if status_code in [status.HTTP_200_OK, status.HTTP_201_CREATED]: data = [u.username for u in team.user_set.all()] return Response(data, status=status_code) - @action(methods=['POST'], detail=True) + @action(methods=["POST"], detail=True) def share(self, request, *args, **kwargs): + """ + Performs sharing a team project operations. + """ + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() - data = merge_dicts(request.data.items(), {'team': self.object.pk}) + data = merge_dicts(request.data.items(), {"team": self.object.pk}) remove = data.get("remove") if remove and remove is not isinstance(remove, bool): @@ -91,7 +107,6 @@ def share(self, request, *args, **kwargs): if serializer.is_valid(): serializer.save() else: - return Response(data=serializer.errors, - status=status.HTTP_400_BAD_REQUEST) + return Response(data=serializer.errors, status=status.HTTP_400_BAD_REQUEST) return Response(status=status.HTTP_204_NO_CONTENT) diff --git a/onadata/apps/api/viewsets/user_profile_viewset.py b/onadata/apps/api/viewsets/user_profile_viewset.py index 3aa8b18e12..f02ffa851a 100644 --- a/onadata/apps/api/viewsets/user_profile_viewset.py +++ b/onadata/apps/api/viewsets/user_profile_viewset.py @@ -5,19 +5,19 @@ import datetime import json -from future.moves.urllib.parse import urlencode - -from past.builtins import basestring # pylint: disable=redefined-builtin from django.conf import settings from django.core.cache import cache from django.core.validators import ValidationError from django.db.models import Count -from django.http import HttpResponseRedirect, HttpResponseBadRequest -from django.utils.translation import ugettext as _ +from django.http import HttpResponseBadRequest, HttpResponseRedirect from django.utils import timezone from django.utils.module_loading import import_string +from django.utils.translation import gettext as _ + +from six.moves.urllib.parse import urlencode +from multidb.pinning import use_master from registration.models import RegistrationProfile from rest_framework import serializers, status from rest_framework.decorators import action @@ -26,35 +26,33 @@ from rest_framework.generics import get_object_or_404 from rest_framework.response import Response from rest_framework.viewsets import ModelViewSet -from multidb.pinning import use_master -from onadata.apps.api.tasks import send_verification_email from onadata.apps.api.permissions import UserProfilePermissions +from onadata.apps.api.tasks import send_verification_email from onadata.apps.api.tools import get_baseviewset_class from onadata.apps.logger.models.instance import Instance from onadata.apps.main.models import UserProfile -from onadata.libs.utils.email import (get_verification_email_data, - get_verification_url) -from onadata.libs.utils.cache_tools import (safe_delete, - CHANGE_PASSWORD_ATTEMPTS, - LOCKOUT_CHANGE_PASSWORD_USER, - USER_PROFILE_PREFIX) from onadata.libs import filters -from onadata.libs.utils.user_auth import invalidate_and_regen_tokens -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.mixins.object_lookup_mixin import ObjectLookupMixin -from onadata.libs.serializers.monthly_submissions_serializer import \ - MonthlySubmissionsSerializer -from onadata.libs.serializers.user_profile_serializer import \ - UserProfileSerializer +from onadata.libs.serializers.monthly_submissions_serializer import ( + MonthlySubmissionsSerializer, +) +from onadata.libs.serializers.user_profile_serializer import UserProfileSerializer +from onadata.libs.utils.cache_tools import ( + CHANGE_PASSWORD_ATTEMPTS, + LOCKOUT_CHANGE_PASSWORD_USER, + USER_PROFILE_PREFIX, + safe_delete, +) +from onadata.libs.utils.email import get_verification_email_data, get_verification_url +from onadata.libs.utils.user_auth import invalidate_and_regen_tokens BaseViewset = get_baseviewset_class() # pylint: disable=invalid-name -LOCKOUT_TIME = getattr(settings, 'LOCKOUT_TIME', 1800) -MAX_CHANGE_PASSWORD_ATTEMPTS = getattr( - settings, 'MAX_CHANGE_PASSWORD_ATTEMPTS', 10) +LOCKOUT_TIME = getattr(settings, "LOCKOUT_TIME", 1800) +MAX_CHANGE_PASSWORD_ATTEMPTS = getattr(settings, "MAX_CHANGE_PASSWORD_ATTEMPTS", 10) def replace_key_value(lookup, new_value, expected_dict): @@ -80,9 +78,9 @@ def check_if_key_exists(a_key, expected_dict): for key, value in expected_dict.items(): if key == a_key: return True - elif isinstance(value, dict): + if isinstance(value, dict): return check_if_key_exists(a_key, value) - elif isinstance(value, list): + if isinstance(value, list): for list_item in value: if isinstance(list_item, dict): return check_if_key_exists(a_key, list_item) @@ -101,31 +99,35 @@ def serializer_from_settings(): def set_is_email_verified(profile, is_email_verified): - profile.metadata.update({'is_email_verified': is_email_verified}) + """Sets is_email_verified value in the profile's metadata object.""" + profile.metadata.update({"is_email_verified": is_email_verified}) profile.save() def check_user_lockout(request): + """Returns the error object with lockout error message.""" username = request.user.username - lockout = cache.get('{}{}'.format(LOCKOUT_CHANGE_PASSWORD_USER, username)) - response_obj = { - 'error': 'Too many password reset attempts, Try again in {} minutes'} + lockout = cache.get(f"{LOCKOUT_CHANGE_PASSWORD_USER}{username}") if lockout: - time_locked_out = \ - datetime.datetime.now() - datetime.datetime.strptime( - lockout, '%Y-%m-%dT%H:%M:%S') - remaining_time = round( - (LOCKOUT_TIME - - time_locked_out.seconds) / 60) - response = response_obj['error'].format(remaining_time) - return response + time_locked_out = datetime.datetime.now() - datetime.datetime.strptime( + lockout, "%Y-%m-%dT%H:%M:%S" + ) + remaining_time = round((LOCKOUT_TIME - time_locked_out.seconds) / 60) + response_obj = { + "error": _( + "Too many password reset attempts. " + f"Try again in {remaining_time} minutes" + ) + } + return response_obj + return None def change_password_attempts(request): """Track number of login attempts made by user within a specified amount - of time""" + of time""" username = request.user.username - password_attempts = '{}{}'.format(CHANGE_PASSWORD_ATTEMPTS, username) + password_attempts = f"{CHANGE_PASSWORD_ATTEMPTS}{username}" attempts = cache.get(password_attempts) if attempts: @@ -133,9 +135,10 @@ def change_password_attempts(request): attempts = cache.get(password_attempts) if attempts >= MAX_CHANGE_PASSWORD_ATTEMPTS: cache.set( - '{}{}'.format(LOCKOUT_CHANGE_PASSWORD_USER, username), - datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'), - LOCKOUT_TIME) + f"{LOCKOUT_CHANGE_PASSWORD_USER}{username}", + datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S"), + LOCKOUT_TIME, + ) if check_user_lockout(request): return check_user_lockout(request) @@ -146,30 +149,34 @@ def change_password_attempts(request): return 1 +# pylint: disable=too-many-ancestors class UserProfileViewSet( - AuthenticateHeaderMixin, # pylint: disable=R0901 - CacheControlMixin, - ETagsMixin, - ObjectLookupMixin, - BaseViewset, - ModelViewSet): + AuthenticateHeaderMixin, + CacheControlMixin, + ETagsMixin, + ObjectLookupMixin, + BaseViewset, + ModelViewSet, +): """ List, Retrieve, Update, Create/Register users. """ - queryset = UserProfile.objects.select_related().filter( - user__is_active=True).exclude( - user__username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME) + + queryset = ( + UserProfile.objects.select_related() + .filter(user__is_active=True) + .exclude(user__username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME) + ) serializer_class = serializer_from_settings() - lookup_field = 'user' + lookup_field = "user" permission_classes = [UserProfilePermissions] filter_backends = (filters.UserProfileFilter, OrderingFilter) - ordering = ('user__username', ) + ordering = ("user__username",) def get_object(self, queryset=None): """Lookup user profile by pk or username""" if self.kwargs.get(self.lookup_field, None) is None: - raise ParseError( - 'Expected URL keyword argument `%s`.' % self.lookup_field) + raise ParseError(_(f"Expected URL keyword argument `{self.lookup_field}`.")) if queryset is None: queryset = self.filter_queryset(self.get_queryset()) @@ -180,7 +187,7 @@ def get_object(self, queryset=None): if self.lookup_field in serializer.get_fields(): k = serializer.get_fields()[self.lookup_field] if isinstance(k, serializers.HyperlinkedRelatedField): - lookup_field = '%s__%s' % (self.lookup_field, k.lookup_field) + lookup_field = f"{self.lookup_field}__{k.lookup_field}" lookup = self.kwargs[self.lookup_field] filter_kwargs = {lookup_field: lookup} @@ -188,9 +195,9 @@ def get_object(self, queryset=None): try: user_pk = int(lookup) except (TypeError, ValueError): - filter_kwargs = {'%s__iexact' % lookup_field: lookup} + filter_kwargs = {f"{lookup_field}__iexact": lookup} else: - filter_kwargs = {'user__pk': user_pk} + filter_kwargs = {"user__pk": user_pk} obj = get_object_or_404(queryset, **filter_kwargs) @@ -200,84 +207,77 @@ def get_object(self, queryset=None): return obj def update(self, request, *args, **kwargs): - """ Update user in cache and db""" - username = kwargs.get('user') - response = super(UserProfileViewSet, self)\ - .update(request, *args, **kwargs) - cache.set(f'{USER_PROFILE_PREFIX}{username}', response.data) + """Update user in cache and db""" + username = kwargs.get("user") + response = super().update(request, *args, **kwargs) + cache.set(f"{USER_PROFILE_PREFIX}{username}", response.data) return response def retrieve(self, request, *args, **kwargs): - """ Get user profile from cache or db """ - username = kwargs.get('user') - cached_user = cache.get(f'{USER_PROFILE_PREFIX}{username}') + """Get user profile from cache or db""" + username = kwargs.get("user") + cached_user = cache.get(f"{USER_PROFILE_PREFIX}{username}") if cached_user: return Response(cached_user) - response = super(UserProfileViewSet, self)\ - .retrieve(request, *args, **kwargs) + response = super().retrieve(request, *args, **kwargs) return response def create(self, request, *args, **kwargs): - """ Create and cache user profile """ - response = super(UserProfileViewSet, self)\ - .create(request, *args, **kwargs) + """Create and cache user profile""" + response = super().create(request, *args, **kwargs) profile = response.data - user_name = profile.get('username') - cache.set(f'{USER_PROFILE_PREFIX}{user_name}', profile) + user_name = profile.get("username") + cache.set(f"{USER_PROFILE_PREFIX}{user_name}", profile) return response - @action(methods=['POST'], detail=True) + @action(methods=["POST"], detail=True) def change_password(self, request, *args, **kwargs): # noqa """ Change user's password. """ # clear cache - safe_delete(f'{USER_PROFILE_PREFIX}{request.user.username}') + safe_delete(f"{USER_PROFILE_PREFIX}{request.user.username}") user_profile = self.get_object() - current_password = request.data.get('current_password', None) - new_password = request.data.get('new_password', None) + current_password = request.data.get("current_password", None) + new_password = request.data.get("new_password", None) lock_out = check_user_lockout(request) - response_obj = { - 'error': 'Invalid password. You have {} attempts left.'} if new_password: if not lock_out: if user_profile.user.check_password(current_password): - data = { - 'username': user_profile.user.username - } + data = {"username": user_profile.user.username} metadata = user_profile.metadata or {} - metadata['last_password_edit'] = timezone.now().isoformat() + metadata["last_password_edit"] = timezone.now().isoformat() user_profile.user.set_password(new_password) user_profile.metadata = metadata user_profile.user.save() user_profile.save() - data.update(invalidate_and_regen_tokens( - user=user_profile.user)) + data.update(invalidate_and_regen_tokens(user=user_profile.user)) - return Response( - status=status.HTTP_200_OK, data=data) + return Response(status=status.HTTP_200_OK, data=data) response = change_password_attempts(request) if isinstance(response, int): - limits_remaining = \ - MAX_CHANGE_PASSWORD_ATTEMPTS - response - response = response_obj['error'].format( - limits_remaining) - return Response(data=response, - status=status.HTTP_400_BAD_REQUEST) + limits_remaining = MAX_CHANGE_PASSWORD_ATTEMPTS - response + response = { + "error": _( + "Invalid password. " + f"You have {limits_remaining} attempts left." + ) + } + return Response(data=response, status=status.HTTP_400_BAD_REQUEST) return Response(data=lock_out, status=status.HTTP_400_BAD_REQUEST) def partial_update(self, request, *args, **kwargs): + """Allows for partial update of the user profile data.""" profile = self.get_object() metadata = profile.metadata or {} - if request.data.get('overwrite') == 'false': - if isinstance(request.data.get('metadata'), basestring): - metadata_items = json.loads( - request.data.get('metadata')).items() + if request.data.get("overwrite") == "false": + if isinstance(request.data.get("metadata"), str): + metadata_items = json.loads(request.data.get("metadata")).items() else: - metadata_items = request.data.get('metadata').items() + metadata_items = request.data.get("metadata").items() for key, value in metadata_items: if check_if_key_exists(key, metadata): @@ -289,26 +289,24 @@ def partial_update(self, request, *args, **kwargs): profile.save() return Response(data=profile.metadata, status=status.HTTP_200_OK) - return super(UserProfileViewSet, self).partial_update( - request, *args, **kwargs) + return super().partial_update(request, *args, **kwargs) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def monthly_submissions(self, request, *args, **kwargs): - """ Get the total number of submissions for a user """ + """Get the total number of submissions for a user""" # clear cache - safe_delete(f'{USER_PROFILE_PREFIX}{request.user.username}') + safe_delete(f"{USER_PROFILE_PREFIX}{request.user.username}") profile = self.get_object() - month_param = self.request.query_params.get('month', None) - year_param = self.request.query_params.get('year', None) + month_param = self.request.query_params.get("month", None) + year_param = self.request.query_params.get("year", None) # check if parameters are valid if month_param: - if not month_param.isdigit() or \ - int(month_param) not in range(1, 13): - raise ValidationError(u'Invalid month provided as parameter') + if not month_param.isdigit() or int(month_param) not in range(1, 13): + raise ValidationError("Invalid month provided as parameter") if year_param: if not year_param.isdigit() or len(year_param) != 4: - raise ValidationError(u'Invalid year provided as parameter') + raise ValidationError("Invalid year provided as parameter") # Use query parameter values for month and year # if none, use the current month and year @@ -316,59 +314,63 @@ def monthly_submissions(self, request, *args, **kwargs): month = month_param if month_param else now.month year = year_param if year_param else now.year - instance_count = Instance.objects.filter( - xform__user=profile.user, - xform__deleted_at__isnull=True, - date_created__year=year, - date_created__month=month).values('xform__shared').annotate( - num_instances=Count('id')) + instance_count = ( + Instance.objects.filter( + xform__user=profile.user, + xform__deleted_at__isnull=True, + date_created__year=year, + date_created__month=month, + ) + .values("xform__shared") + .annotate(num_instances=Count("id")) + ) serializer = MonthlySubmissionsSerializer(instance_count, many=True) return Response(serializer.data[0]) + # pylint: disable=no-self-use @action(detail=False) def verify_email(self, request, *args, **kwargs): + """Accpet's email verification token and marks the profile as verified.""" verified_key_text = getattr(settings, "VERIFIED_KEY_TEXT", None) if not verified_key_text: return Response(status=status.HTTP_204_NO_CONTENT) - redirect_url = request.query_params.get('redirect_url') - verification_key = request.query_params.get('verification_key') + redirect_url = request.query_params.get("redirect_url") + verification_key = request.query_params.get("verification_key") response_message = _("Missing or invalid verification key") if verification_key: - rp = None + registration_profile = None try: - rp = RegistrationProfile.objects.select_related( - 'user', 'user__profile').get( - activation_key=verification_key) + registration_profile = RegistrationProfile.objects.select_related( + "user", "user__profile" + ).get(activation_key=verification_key) except RegistrationProfile.DoesNotExist: with use_master: try: - rp = RegistrationProfile.objects.select_related( - 'user', 'user__profile').get( - activation_key=verification_key) + registration_profile = ( + RegistrationProfile.objects.select_related( + "user", "user__profile" + ).get(activation_key=verification_key) + ) except RegistrationProfile.DoesNotExist: pass - if rp: - rp.activation_key = verified_key_text - rp.save() + if registration_profile: + registration_profile.activation_key = verified_key_text + registration_profile.save() - username = rp.user.username - set_is_email_verified(rp.user.profile, True) + username = registration_profile.user.username + set_is_email_verified(registration_profile.user.profile, True) # Clear profiles cache - safe_delete(f'{USER_PROFILE_PREFIX}{username}') + safe_delete(f"{USER_PROFILE_PREFIX}{username}") - response_data = { - 'username': username, - 'is_email_verified': True - } + response_data = {"username": username, "is_email_verified": True} if redirect_url: query_params_string = urlencode(response_data) - redirect_url = '{}?{}'.format(redirect_url, - query_params_string) + redirect_url = f"{redirect_url}?{query_params_string}" return HttpResponseRedirect(redirect_url) @@ -376,34 +378,44 @@ def verify_email(self, request, *args, **kwargs): return HttpResponseBadRequest(response_message) - @action(methods=['POST'], detail=False) + # pylint: disable=no-self-use + @action(methods=["POST"], detail=False) def send_verification_email(self, request, *args, **kwargs): + """Sends verification email on user profile registration.""" verified_key_text = getattr(settings, "VERIFIED_KEY_TEXT", None) if not verified_key_text: return Response(status=status.HTTP_204_NO_CONTENT) - username = request.data.get('username') - redirect_url = request.data.get('redirect_url') + username = request.data.get("username") + redirect_url = request.data.get("redirect_url") response_message = _("Verification email has NOT been sent") if username: try: - rp = RegistrationProfile.objects.get(user__username=username) + registration_profile = RegistrationProfile.objects.get( + user__username=username + ) except RegistrationProfile.DoesNotExist: pass else: - set_is_email_verified(rp.user.profile, False) + set_is_email_verified(registration_profile.user.profile, False) - verification_key = rp.activation_key + verification_key = registration_profile.activation_key if verification_key == verified_key_text: - verification_key = (rp.user.registrationprofile. - create_new_activation_key()) + verification_key = ( + registration_profile.user.registrationprofile.create_new_activation_key() + ) verification_url = get_verification_url( - redirect_url, request, verification_key) + redirect_url, request, verification_key + ) email_data = get_verification_email_data( - rp.user.email, rp.user.username, verification_url, request) + registration_profile.user.email, + registration_profile.user.username, + verification_url, + request, + ) send_verification_email.delay(**email_data) response_message = _("Verification email has been sent") diff --git a/onadata/apps/api/viewsets/v2/tableau_viewset.py b/onadata/apps/api/viewsets/v2/tableau_viewset.py index 80ad85a0e2..158d0ce055 100644 --- a/onadata/apps/api/viewsets/v2/tableau_viewset.py +++ b/onadata/apps/api/viewsets/v2/tableau_viewset.py @@ -1,4 +1,7 @@ -import json +# -*- coding: utf-8 -*- +""" +Implements the /api/v2/tableau endpoint +""" import re from typing import List @@ -12,27 +15,33 @@ from onadata.apps.logger.models import Instance from onadata.apps.logger.models.xform import XForm from onadata.apps.api.tools import replace_attachment_name_with_url -from onadata.apps.api.viewsets.open_data_viewset import ( - OpenDataViewSet) +from onadata.apps.api.viewsets.open_data_viewset import OpenDataViewSet from onadata.libs.serializers.data_serializer import TableauDataSerializer from onadata.libs.utils.common_tags import ( - ID, MULTIPLE_SELECT_TYPE, REPEAT_SELECT_TYPE, PARENT_TABLE, PARENT_ID) + ID, + MULTIPLE_SELECT_TYPE, + REPEAT_SELECT_TYPE, + PARENT_TABLE, + PARENT_ID, +) -DEFAULT_TABLE_NAME = 'data' -GPS_PARTS = ['latitude', 'longitude', 'altitude', 'precision'] +DEFAULT_TABLE_NAME = "data" +GPS_PARTS = ["latitude", "longitude", "altitude", "precision"] def process_tableau_data( - data, xform, - parent_table: str = None, - parent_id: int = None, - current_table: str = DEFAULT_TABLE_NAME): + data, + xform, + parent_table: str = None, + parent_id: int = None, + current_table: str = DEFAULT_TABLE_NAME, +): result = [] if data: for idx, row in enumerate(data, start=1): flat_dict = defaultdict(list) - row_id = row.get('_id') + row_id = row.get("_id") if not row_id and parent_id: row_id = int(pairing(parent_id, idx)) @@ -45,35 +54,38 @@ def process_tableau_data( for (key, value) in row.items(): qstn = xform.get_element(key) if qstn: - qstn_type = qstn.get('type') - qstn_name = qstn.get('name') + qstn_type = qstn.get("type") + qstn_name = qstn.get("name") prefix_parts = [ - question['name'] for question in qstn.get_lineage() - if question['type'] == 'group' + question["name"] + for question in qstn.get_lineage() + if question["type"] == "group" ] prefix = "_".join(prefix_parts) if qstn_type == REPEAT_SELECT_TYPE: repeat_data = process_tableau_data( - value, xform, + value, + xform, parent_table=current_table, parent_id=row_id, - current_table=qstn_name) - cleaned_data = unpack_repeat_data( - repeat_data, flat_dict) + current_table=qstn_name, + ) + cleaned_data = unpack_repeat_data(repeat_data, flat_dict) flat_dict[qstn_name] = cleaned_data elif qstn_type == MULTIPLE_SELECT_TYPE: picked_choices = value.split(" ") choice_names = [ - question["name"] for question in qstn["children"]] - list_name = qstn.get('list_name') + question["name"] for question in qstn["children"] + ] + list_name = qstn.get("list_name") select_multiple_data = unpack_select_multiple_data( - picked_choices, list_name, choice_names, prefix) + picked_choices, list_name, choice_names, prefix + ) flat_dict.update(select_multiple_data) - elif qstn_type == 'geopoint': - gps_parts = unpack_gps_data( - value, qstn_name, prefix) + elif qstn_type == "geopoint": + gps_parts = unpack_gps_data(value, qstn_name, prefix) flat_dict.update(gps_parts) else: if prefix: @@ -83,14 +95,13 @@ def process_tableau_data( return result -def unpack_select_multiple_data(picked_choices, list_name, - choice_names, prefix): +def unpack_select_multiple_data(picked_choices, list_name, choice_names, prefix): unpacked_data = {} for choice in choice_names: qstn_name = f"{list_name}_{choice}" if prefix: - qstn_name = prefix + '_' + qstn_name + qstn_name = prefix + "_" + qstn_name if choice in picked_choices: unpacked_data[qstn_name] = "TRUE" @@ -117,16 +128,15 @@ def unpack_repeat_data(repeat_data, flat_dict): def unpack_gps_data(value, qstn_name, prefix): - value_parts = value.split(' ') + value_parts = value.split(" ") gps_xpath_parts = [] for part in GPS_PARTS: name = f"_{qstn_name}_{part}" if prefix: - name = prefix + '_' + name + name = prefix + "_" + name gps_xpath_parts.append((name, None)) if len(value_parts) == 4: - gps_parts = dict( - zip(dict(gps_xpath_parts), value_parts)) + gps_parts = dict(zip(dict(gps_xpath_parts), value_parts)) return gps_parts @@ -135,9 +145,9 @@ def clean_xform_headers(headers: list) -> list: for header in headers: if re.search(r"\[+\d+\]", header): repeat_count = len(re.findall(r"\[+\d+\]", header)) - header = header.split('/')[repeat_count].replace('[1]', '') + header = header.split("/")[repeat_count].replace("[1]", "") - if not header.endswith('gps'): + if not header.endswith("gps"): # Replace special character with underscore header = re.sub(r"\W", r"_", header) ret.append(header) @@ -145,16 +155,16 @@ def clean_xform_headers(headers: list) -> list: class TableauViewSet(OpenDataViewSet): - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def data(self, request, **kwargs): self.object = self.get_object() # get greater than value and cast it to an int - gt_id = request.query_params.get('gt_id') + gt_id = request.query_params.get("gt_id") gt_id = gt_id and parse_int(gt_id) - count = request.query_params.get('count') + count = request.query_params.get("count") pagination_keys = [ self.paginator.page_query_param, - self.paginator.page_size_query_param + self.paginator.page_size_query_param, ] query_param_keys = request.query_params should_paginate = any([k in query_param_keys for k in pagination_keys]) @@ -166,19 +176,23 @@ def data(self, request, **kwargs): xform = self.object.content_object if xform.is_merged_dataset: - qs_kwargs = {'xform_id__in': list( - xform.mergedxform.xforms.values_list('pk', flat=True))} + qs_kwargs = { + "xform_id__in": list( + xform.mergedxform.xforms.values_list("pk", flat=True) + ) + } else: - qs_kwargs = {'xform_id': xform.pk} + qs_kwargs = {"xform_id": xform.pk} if gt_id: - qs_kwargs.update({'id__gt': gt_id}) + qs_kwargs.update({"id__gt": gt_id}) # Filter out deleted submissions instances = Instance.objects.filter( - **qs_kwargs, deleted_at__isnull=True).order_by('pk') + **qs_kwargs, deleted_at__isnull=True + ).order_by("pk") if count: - return Response({'count': instances.count()}) + return Response({"count": instances.count()}) if should_paginate: instances = self.paginate_queryset(instances) @@ -186,7 +200,8 @@ def data(self, request, **kwargs): data = replace_attachment_name_with_url(instances) data = process_tableau_data( - TableauDataSerializer(data, many=True).data, xform) + TableauDataSerializer(data, many=True).data, xform + ) return self.get_streaming_response(data) @@ -194,53 +209,52 @@ def data(self, request, **kwargs): # pylint: disable=arguments-differ def flatten_xform_columns( - self, json_of_columns_fields, table: str = None, - field_prefix: str = None): - ''' + self, json_of_columns_fields, table: str = None, field_prefix: str = None + ): + """ Flattens a json of column fields while splitting columns into separate table names for each repeat - ''' + """ ret = defaultdict(list) for field in json_of_columns_fields: table_name = table or DEFAULT_TABLE_NAME prefix = field_prefix or "" - field_type = field.get('type') + field_type = field.get("type") - if field_type in [REPEAT_SELECT_TYPE, 'group']: - if field_type == 'repeat': - table_name = field.get('name') + if field_type in [REPEAT_SELECT_TYPE, "group"]: + if field_type == "repeat": + table_name = field.get("name") else: prefix = prefix + f"{field['name']}_" columns = self.flatten_xform_columns( - field.get('children'), table=table_name, - field_prefix=prefix) + field.get("children"), table=table_name, field_prefix=prefix + ) for key in columns.keys(): ret[key].extend(columns[key]) elif field_type == MULTIPLE_SELECT_TYPE: - for option in field.get('children'): - list_name = field.get('list_name') - option_name = option.get('name') + for option in field.get("children"): + list_name = field.get("list_name") + option_name = option.get("name") ret[table_name].append( { - 'name': f'{prefix}{list_name}_{option_name}', - 'type': self.get_tableau_type('text') + "name": f"{prefix}{list_name}_{option_name}", + "type": self.get_tableau_type("text"), } ) - elif field_type == 'geopoint': + elif field_type == "geopoint": for part in GPS_PARTS: name = f'_{field["name"]}_{part}' if prefix: name = prefix + name - ret[table_name].append({ - 'name': name, - 'type': self.get_tableau_type(field.get('type')) - }) + ret[table_name].append( + {"name": name, "type": self.get_tableau_type(field.get("type"))} + ) else: ret[table_name].append( { - 'name': prefix + field.get('name'), - 'type': self.get_tableau_type(field.get('type')) + "name": prefix + field.get("name"), + "type": self.get_tableau_type(field.get("type")), } ) return ret @@ -252,29 +266,25 @@ def get_tableau_column_headers(self): tableau_column_headers = defaultdict(list) for table, columns in self.flattened_dict.items(): # Add ID Fields - tableau_column_headers[table].append({ - 'id': ID, - 'dataType': 'int', - 'alias': ID - }) + tableau_column_headers[table].append( + {"id": ID, "dataType": "int", "alias": ID} + ) if table != DEFAULT_TABLE_NAME: - tableau_column_headers[table].append({ - 'id': PARENT_ID, - 'dataType': 'int', - 'alias': PARENT_ID - }) - tableau_column_headers[table].append({ - 'id': PARENT_TABLE, - 'dataType': 'string', - 'alias': PARENT_TABLE - }) + tableau_column_headers[table].append( + {"id": PARENT_ID, "dataType": "int", "alias": PARENT_ID} + ) + tableau_column_headers[table].append( + {"id": PARENT_TABLE, "dataType": "string", "alias": PARENT_TABLE} + ) for column in columns: - tableau_column_headers[table].append({ - 'id': column.get('name'), - 'dataType': column.get('type'), - 'alias': column.get('name') - }) + tableau_column_headers[table].append( + { + "id": column.get("name"), + "dataType": column.get("type"), + "alias": column.get("name"), + } + ) return tableau_column_headers def get_tableau_table_schemas(self) -> List[dict]: @@ -284,23 +294,22 @@ def get_tableau_table_schemas(self) -> List[dict]: column_headers = self.get_tableau_column_headers() for table_name, headers in column_headers.items(): table_schema = {} - table_schema['table_alias'] = table_name - table_schema['connection_name'] = f"{project}_{id_str}" - table_schema['column_headers'] = headers + table_schema["table_alias"] = table_name + table_schema["connection_name"] = f"{project}_{id_str}" + table_schema["column_headers"] = headers if table_name != DEFAULT_TABLE_NAME: - table_schema['connection_name'] += f"_{table_name}" + table_schema["connection_name"] += f"_{table_name}" ret.append(table_schema) return ret - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def schema(self, request, **kwargs): self.object = self.get_object() if isinstance(self.object.content_object, XForm): self.xform = self.object.content_object - xform_json = json.loads(self.xform.json) + xform_json = self.xform.json_dict() headers = self.xform.get_headers(repeat_iterations=1) - self.flattened_dict = self.flatten_xform_columns( - xform_json.get('children')) + self.flattened_dict = self.flatten_xform_columns(xform_json.get("children")) self.xform_headers = clean_xform_headers(headers) data = self.get_tableau_table_schemas() return Response(data=data, status=status.HTTP_200_OK) diff --git a/onadata/apps/api/viewsets/widget_viewset.py b/onadata/apps/api/viewsets/widget_viewset.py index a82aac8a34..8c28a3bc70 100644 --- a/onadata/apps/api/viewsets/widget_viewset.py +++ b/onadata/apps/api/viewsets/widget_viewset.py @@ -1,5 +1,5 @@ from django.shortcuts import get_object_or_404 -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.contrib.contenttypes.models import ContentType from rest_framework.viewsets import ModelViewSet diff --git a/onadata/apps/api/viewsets/xform_submission_viewset.py b/onadata/apps/api/viewsets/xform_submission_viewset.py index f663f604e5..c05d3df4f8 100644 --- a/onadata/apps/api/viewsets/xform_submission_viewset.py +++ b/onadata/apps/api/viewsets/xform_submission_viewset.py @@ -4,7 +4,7 @@ """ from django.conf import settings from django.http import UnreadablePostError -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import mixins, permissions, status, viewsets from rest_framework.authentication import (BasicAuthentication, @@ -85,8 +85,8 @@ def get_serializer_class(self): content_type = self.request.content_type.lower() if 'application/json' in content_type: - if 'RapidProMailroom' in self.request.META.get( - 'HTTP_USER_AGENT', ''): + if 'RapidProMailroom' in self.request.headers.get( + 'User-Agent', ''): return RapidProJSONSubmissionSerializer self.request.accepted_renderer = JSONRenderer() diff --git a/onadata/apps/api/viewsets/xform_viewset.py b/onadata/apps/api/viewsets/xform_viewset.py index ea4efecfa9..a55135c8e2 100644 --- a/onadata/apps/api/viewsets/xform_viewset.py +++ b/onadata/apps/api/viewsets/xform_viewset.py @@ -1,10 +1,14 @@ +# -*- coding: utf-8 -*- +""" +The /forms API endpoint. +""" import json import os import random from datetime import datetime from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.exceptions import ValidationError from django.core.files.base import ContentFile from django.core.files.storage import default_storage @@ -17,21 +21,21 @@ HttpResponseRedirect, StreamingHttpResponse, ) -from django.utils import six, timezone +from django.shortcuts import get_object_or_404 +from django.utils import timezone from django.utils.http import urlencode -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.views.decorators.cache import never_cache -from django.shortcuts import get_object_or_404 + +import six from django_filters.rest_framework import DjangoFilterBackend -from future.moves.urllib.parse import urlparse +from six.moves.urllib.parse import urlparse try: from multidb.pinning import use_master except ImportError: pass -from onadata.apps.messaging.constants import XFORM, FORM_UPDATED -from onadata.apps.messaging.serializers import send_message from pyxform.builder import create_survey_element_from_dict from pyxform.xls2json import parse_file_to_json from rest_framework import exceptions, status @@ -41,15 +45,18 @@ from rest_framework.settings import api_settings from rest_framework.viewsets import ModelViewSet -from onadata.apps.api import tools as utils from onadata.apps.api import tasks +from onadata.apps.api import tools as utils from onadata.apps.api.permissions import XFormPermissions from onadata.apps.api.tools import get_baseviewset_class from onadata.apps.logger.models.xform import XForm, XFormUserObjectPermission from onadata.apps.logger.models.xform_version import XFormVersion from onadata.apps.logger.xform_instance_parser import XLSFormError +from onadata.apps.messaging.constants import FORM_UPDATED, XFORM +from onadata.apps.messaging.serializers import send_message from onadata.apps.viewer.models.export import Export from onadata.libs import authentication, filters +from onadata.libs.exceptions import EnketoError from onadata.libs.mixins.anonymous_user_public_forms_mixin import ( AnonymousUserPublicFormsMixin, ) @@ -72,25 +79,24 @@ process_async_export, response_for_format, ) +from onadata.libs.utils.cache_tools import PROJ_OWNER_CACHE, safe_delete from onadata.libs.utils.common_tools import json_stream from onadata.libs.utils.csv_import import ( get_async_csv_submission_status, + submission_xls_to_csv, submit_csv, submit_csv_async, - submission_xls_to_csv, ) from onadata.libs.utils.export_tools import parse_request_export_options from onadata.libs.utils.logger_tools import publish_form from onadata.libs.utils.model_tools import queryset_iterator from onadata.libs.utils.string import str2bool from onadata.libs.utils.viewer_tools import ( - get_enketo_urls, generate_enketo_form_defaults, + get_enketo_urls, get_form_url, ) -from onadata.libs.exceptions import EnketoError -from onadata.settings.common import XLS_EXTENSIONS, CSV_EXTENSION -from onadata.libs.utils.cache_tools import PROJ_OWNER_CACHE, safe_delete +from onadata.settings.common import CSV_EXTENSION, XLS_EXTENSIONS ENKETO_AUTH_COOKIE = getattr(settings, "ENKETO_AUTH_COOKIE", "__enketo") ENKETO_META_UID_COOKIE = getattr( @@ -100,14 +106,18 @@ settings, "ENKETO_META_USERNAME_COOKIE", "__enketo_meta_username" ) +# pylint: disable=invalid-name BaseViewset = get_baseviewset_class() +User = get_user_model() def upload_to_survey_draft(filename, username): + """Return the ``filename`` in the ``username`` survey-drafts directory.""" return os.path.join(username, "survey-drafts", os.path.split(filename)[1]) def get_survey_dict(csv_name): + """Returns the a CSV XLSForm file into a python object.""" survey_file = default_storage.open(csv_name, "rb") survey_dict = parse_file_to_json(survey_file.name, default_name="data") @@ -118,7 +128,7 @@ def get_survey_dict(csv_name): def _get_user(username): users = User.objects.filter(username__iexact=username) - return users.count() and users[0] or None + return users[0] if users.count() else None def _get_owner(request): @@ -128,14 +138,15 @@ def _get_owner(request): owner_obj = _get_user(owner) if owner_obj is None: - raise ValidationError("User with username %s does not exist." % owner) - else: - owner = owner_obj + raise ValidationError(f"User with username {owner} does not exist.") + owner = owner_obj return owner def value_for_type(form, field, value): + """Returns a boolean value for the ``field`` of type ``BooleanField`` otherwise + returns the same ``value`` back.""" if form._meta.get_field(field).get_internal_type() == "BooleanField": return str2bool(value) @@ -163,10 +174,12 @@ def _try_update_xlsform(request, xform, owner): def result_has_error(result): + """Returns True if the ``result`` is a dict and has a type.""" return isinstance(result, dict) and result.get("type") def get_survey_xml(csv_name): + """Creates and returns the XForm XML from a CSV XLSform.""" survey_dict = get_survey_dict(csv_name) survey = create_survey_element_from_dict(survey_dict) return survey.to_xml() @@ -175,7 +188,7 @@ def get_survey_xml(csv_name): def set_enketo_signed_cookies(resp, username=None, json_web_token=None): """Set signed cookies for JWT token in the HTTPResponse resp object.""" if not username and not json_web_token: - return + return None max_age = 30 * 24 * 60 * 60 * 1000 enketo_meta_uid = {"max_age": max_age, "salt": settings.ENKETO_API_SALT} @@ -202,6 +215,7 @@ def parse_webform_return_url(return_url, request): this data or data in the request. Construct a proper return URL, which has stripped the authentication data, to return the user. """ + jwt_param = None url = urlparse(return_url) try: @@ -210,17 +224,17 @@ def parse_webform_return_url(return_url, request): jwt_param = jwt_param and jwt_param[0].split("=")[1] if not jwt_param: - return + return None except IndexError: pass if "/_/" in return_url: # offline url - redirect_url = "%s://%s%s#%s" % (url.scheme, url.netloc, url.path, url.fragment) + redirect_url = f"{url.scheme}://{url.netloc}{url.path}#{url.fragment}" elif "/::" in return_url: # non-offline url - redirect_url = "%s://%s%s" % (url.scheme, url.netloc, url.path) + redirect_url = f"{url.scheme}://{url.netloc}{url.path}" else: # unexpected format - return + return None response_redirect = HttpResponseRedirect(redirect_url) @@ -241,8 +255,10 @@ def parse_webform_return_url(return_url, request): ) return response_redirect + return None +# pylint: disable=too-many-ancestors class XFormViewSet( AnonymousUserPublicFormsMixin, CacheControlMixin, @@ -339,9 +355,11 @@ def get_serializer_class(self): if self.action == "list": return XFormBaseSerializer - return super(XFormViewSet, self).get_serializer_class() + return super().get_serializer_class() + # pylint: disable=unused-argument def create(self, request, *args, **kwargs): + """Support XLSForm publishing endpoint `POST /api/v1/forms`.""" try: owner = _get_owner(request) except ValidationError as e: @@ -362,6 +380,7 @@ def create(self, request, *args, **kwargs): return Response(survey, status=status.HTTP_400_BAD_REQUEST) + # pylint: disable=unused-argument @action(methods=["POST", "GET"], detail=False) def create_async(self, request, *args, **kwargs): """Temporary Endpoint for Async form creation""" @@ -369,7 +388,8 @@ def create_async(self, request, *args, **kwargs): resp_code = status.HTTP_400_BAD_REQUEST if request.method == "GET": - self.etag_data = "{}".format(timezone.now()) + # pylint: disable=attribute-defined-outside-init + self.etag_data = f"{timezone.now()}" survey = tasks.get_async_status(request.query_params.get("job_uuid")) if "pk" in survey: @@ -414,21 +434,27 @@ def create_async(self, request, *args, **kwargs): @action(methods=["GET", "HEAD"], detail=True) @never_cache - def form(self, request, format="json", **kwargs): + def form(self, request, **kwargs): + """Returns the XLSForm in any of JSON, XML, XLS(X), CSV formats.""" form = self.get_object() - if format not in ["json", "xml", "xls", "csv"]: + form_format = kwargs.get("format", "json") + if form_format not in ["json", "xml", "xls", "xlsx", "csv"]: return HttpResponseBadRequest( "400 BAD REQUEST", content_type="application/json", status=400 ) - self.etag_data = "{}".format(form.date_modified) - filename = form.id_string + "." + format - response = response_for_format(form, format=format) + # pylint: disable=attribute-defined-outside-init + self.etag_data = f"{form.date_modified}" + filename = form.id_string + "." + form_format + response = response_for_format(form, format=form_format) response["Content-Disposition"] = "attachment; filename=" + filename return response + # pylint: disable=no-self-use + # pylint: disable=unused-argument @action(methods=["GET"], detail=False) def login(self, request, **kwargs): + """Authenticate and redirect to URL in `return` query parameter.""" return_url = request.query_params.get("return") if return_url: @@ -447,10 +473,12 @@ def login(self, request, **kwargs): return HttpResponseForbidden("Authentication failure, cannot redirect") + # pylint: disable=unused-argument @action(methods=["GET"], detail=True) def enketo(self, request, **kwargs): """Expose enketo urls.""" survey_type = self.kwargs.get("survey_type") or request.GET.get("survey_type") + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() form_url = get_form_url( request, @@ -472,7 +500,7 @@ def enketo(self, request, **kwargs): preview_url = enketo_urls.get("preview_url") single_submit_url = enketo_urls.get("single_url") except EnketoError as e: - data = {"message": _("Enketo error: %s" % e)} + data = {"message": _(f"Enketo error: {e}")} else: if survey_type == "single": http_status = status.HTTP_200_OK @@ -487,8 +515,10 @@ def enketo(self, request, **kwargs): return Response(data, http_status) + # pylint: disable=unused-argument @action(methods=["POST", "GET"], detail=False) def survey_preview(self, request, **kwargs): + """Handle survey preview XLSForms.""" username = request.user.username if request.method.upper() == "POST": if not username: @@ -496,9 +526,10 @@ def survey_preview(self, request, **kwargs): csv_data = request.data.get("body") if csv_data: - rand_name = "survey_draft_%s.csv" % "".join( + random_name = "".join( random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6) ) + rand_name = f"survey_draft_{random_name}.csv" csv_file = ContentFile(csv_data) csv_name = default_storage.save( upload_to_survey_draft(rand_name, username), csv_file @@ -512,34 +543,33 @@ def survey_preview(self, request, **kwargs): return Response( {"unique_string": rand_name, "username": username}, status=200 ) - else: - raise ParseError("Missing body") - - if request.method.upper() == "GET": - filename = request.query_params.get("filename") - username = request.query_params.get("username") - - if not username: - raise ParseError("Username not provided") - if not filename: - raise ParseError("Filename MUST be provided") + raise ParseError("Missing body") - csv_name = upload_to_survey_draft(filename, username) + filename = request.query_params.get("filename") + username = request.query_params.get("username") - result = publish_form(lambda: get_survey_xml(csv_name)) + if not username: + raise ParseError("Username not provided") + if not filename: + raise ParseError("Filename MUST be provided") - if result_has_error(result): - raise ParseError(result.get("text")) + csv_name = upload_to_survey_draft(filename, username) + result = publish_form(lambda: get_survey_xml(csv_name)) + if result_has_error(result): + raise ParseError(result.get("text")) - self.etag_data = result + # pylint: disable=attribute-defined-outside-init + self.etag_data = result - return Response(result, status=200) + return Response(result, status=200) def retrieve(self, request, *args, **kwargs): + """Returns a forms properties.""" lookup_field = self.lookup_field lookup = self.kwargs.get(lookup_field) if lookup == self.public_forms_endpoint: + # pylint: disable=attribute-defined-outside-init self.object_list = self._get_public_forms_queryset() page = self.paginate_queryset(self.object_list) @@ -558,12 +588,14 @@ def retrieve(self, request, *args, **kwargs): if export_type is None or export_type in ["json", "debug"]: # perform default viewset retrieve, no data export - return super(XFormViewSet, self).retrieve(request, *args, **kwargs) + return super().retrieve(request, *args, **kwargs) return custom_response_handler(request, xform, query, export_type, token, meta) @action(methods=["POST"], detail=True) def share(self, request, *args, **kwargs): + """Perform form sharing.""" + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() usernames_str = request.data.get("usernames", request.data.get("username")) @@ -589,6 +621,8 @@ def share(self, request, *args, **kwargs): @action(methods=["POST"], detail=True) def clone(self, request, *args, **kwargs): + """Clone/duplicate an existing form.""" + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() data = {"xform": self.object.pk, "username": request.data.get("username")} project = request.data.get("project_id") @@ -598,19 +632,20 @@ def clone(self, request, *args, **kwargs): if serializer.is_valid(): clone_to_user = User.objects.get(username=data["username"]) if not request.user.has_perm("can_add_xform", clone_to_user.profile): + user = request.user.username + account = data["username"] raise exceptions.PermissionDenied( detail=_( - "User %(user)s has no permission to add " - "xforms to account %(account)s" - % {"user": request.user.username, "account": data["username"]} + f"User {user} has no permission to add " + f"xforms to account {account}" ) ) try: xform = serializer.save() - except IntegrityError: + except IntegrityError as e: raise ParseError( "A clone with the same id_string has already been created" - ) + ) from e serializer = XFormSerializer( xform.cloned_form, context={"request": request} ) @@ -632,6 +667,7 @@ def data_import(self, request, *args, **kwargs): for POST request passing the `request.FILES.get('xls_file')` upload for import if xls_file is provided instead of csv_file """ + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() resp = {} if request.method == "GET": @@ -642,14 +678,14 @@ def data_import(self, request, *args, **kwargs): ) ) self.last_modified_date = timezone.now() - except ValueError: + except ValueError as e: raise ParseError( ( "The instance of the result is not a " "basestring; the job_uuid variable might " "be incorrect" ) - ) + ) from e else: csv_file = request.FILES.get("csv_file", None) xls_file = request.FILES.get("xls_file", None) @@ -667,7 +703,11 @@ def data_import(self, request, *args, **kwargs): if xls_file and xls_file.name.split(".")[-1] in XLS_EXTENSIONS: csv_file = submission_xls_to_csv(xls_file) overwrite = request.query_params.get("overwrite") - overwrite = True if overwrite and overwrite.lower() == "true" else False + overwrite = ( + overwrite.lower() == "true" + if isinstance(overwrite, str) + else overwrite + ) size_threshold = settings.CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD try: csv_size = csv_file.size @@ -690,8 +730,7 @@ def data_import(self, request, *args, **kwargs): ) if task is None: raise ParseError("Task not found") - else: - resp.update({"task_id": task.task_id}) + resp.update({"task_id": task.task_id}) return Response( data=resp, @@ -710,6 +749,7 @@ def csv_import(self, request, *args, **kwargs): for GET requests passing `job_uuid` query param for job progress polling """ + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() resp = {} if request.method == "GET": @@ -719,15 +759,16 @@ def csv_import(self, request, *args, **kwargs): request.query_params.get("job_uuid") ) ) + # pylint: disable=attribute-defined-outside-init self.last_modified_date = timezone.now() - except ValueError: + except ValueError as e: raise ParseError( ( "The instance of the result is not a " "basestring; the job_uuid variable might " "be incorrect" ) - ) + ) from e else: csv_file = request.FILES.get("csv_file", None) if csv_file is None: @@ -736,7 +777,11 @@ def csv_import(self, request, *args, **kwargs): resp.update({"error": "csv_file not a csv file"}) else: overwrite = request.query_params.get("overwrite") - overwrite = True if overwrite and overwrite.lower() == "true" else False + overwrite = ( + overwrite.lower() == "true" + if isinstance(overwrite, str) + else overwrite + ) size_threshold = settings.CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD if csv_file.size < size_threshold: resp.update( @@ -755,8 +800,7 @@ def csv_import(self, request, *args, **kwargs): ) if task is None: raise ParseError("Task not found") - else: - resp.update({"task_id": task.task_id}) + resp.update({"task_id": task.task_id}) return Response( data=resp, @@ -766,6 +810,8 @@ def csv_import(self, request, *args, **kwargs): ) def partial_update(self, request, *args, **kwargs): + """Partial update of a form's properties.""" + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() owner = self.object.user @@ -776,12 +822,13 @@ def partial_update(self, request, *args, **kwargs): return _try_update_xlsform(request, self.object, owner) try: - return super(XFormViewSet, self).partial_update(request, *args, **kwargs) + return super().partial_update(request, *args, **kwargs) except XLSFormError as e: - raise ParseError(str(e)) + raise ParseError(str(e)) from e @action(methods=["DELETE", "GET"], detail=True) def delete_async(self, request, *args, **kwargs): + """Delete asynchronous endpoint `/api/v1/forms/{pk}/delete_async`.""" if request.method == "DELETE": xform = self.get_object() resp = { @@ -799,11 +846,13 @@ def delete_async(self, request, *args, **kwargs): job_uuid = request.query_params.get("job_uuid") resp = tasks.get_async_status(job_uuid) resp_code = status.HTTP_202_ACCEPTED - self.etag_data = "{}".format(timezone.now()) + # pylint: disable=attribute-defined-outside-init + self.etag_data = f"{timezone.now()}" return Response(data=resp, status=resp_code) def destroy(self, request, *args, **kwargs): + """Soft deletes a form - `DELETE /api/v1/forms/{pk}`""" xform = self.get_object() user = request.user xform.soft_delete(user=user) @@ -812,23 +861,25 @@ def destroy(self, request, *args, **kwargs): @action(methods=["GET"], detail=True) def versions(self, request, *args, **kwargs): + """Returns all form versions.""" xform = self.get_object() version_id = kwargs.get("version_id") requested_format = kwargs.get("format") or "json" - if not version_id: - queryset = XFormVersion.objects.filter(xform=xform) - serializer = XFormVersionListSerializer( - queryset, many=True, context={"request": request} - ) - return Response(data=serializer.data, status=status.HTTP_200_OK) - if version_id: version = get_object_or_404(XFormVersion, version=version_id, xform=xform) return response_for_format(version, format=requested_format) + queryset = XFormVersion.objects.filter(xform=xform) + serializer = XFormVersionListSerializer( + queryset, many=True, context={"request": request} + ) + + return Response(data=serializer.data, status=status.HTTP_200_OK) + @action(methods=["GET"], detail=True) def export_async(self, request, *args, **kwargs): + """Returns the status of an async export.""" job_uuid = request.query_params.get("job_uuid") export_type = request.query_params.get("format") query = request.query_params.get("query") @@ -878,7 +929,8 @@ def export_async(self, request, *args, **kwargs): content_type="application/json", ) - self.etag_data = "{}".format(timezone.now()) + # pylint: disable=attribute-defined-outside-init + self.etag_data = f"{timezone.now()}" return Response( data=resp, status=status.HTTP_202_ACCEPTED, content_type="application/json" @@ -900,6 +952,7 @@ def get_json_string(item): ).data ) + # pylint: disable=http-response-with-content-type-json response = StreamingHttpResponse( json_stream(queryset, get_json_string), content_type="application/json" ) @@ -917,19 +970,21 @@ def get_json_string(item): return response def list(self, request, *args, **kwargs): - STREAM_DATA = getattr(settings, "STREAM_DATA", False) + """List forms API endpoint `GET /api/v1/forms`.""" + stream_data = getattr(settings, "STREAM_DATA", False) try: queryset = self.filter_queryset(self.get_queryset()) last_modified = queryset.values_list("date_modified", flat=True).order_by( "-date_modified" ) + # pylint: disable=attribute-defined-outside-init if last_modified: self.etag_data = last_modified[0].isoformat() - if STREAM_DATA: + if stream_data: self.object_list = queryset resp = self._get_streaming_response() else: - resp = super(XFormViewSet, self).list(request, *args, **kwargs) + resp = super().list(request, *args, **kwargs) except XLSFormError as e: resp = HttpResponseBadRequest(e) diff --git a/onadata/apps/logger/admin.py b/onadata/apps/logger/admin.py index 5366ea069d..d1a14f8cfd 100644 --- a/onadata/apps/logger/admin.py +++ b/onadata/apps/logger/admin.py @@ -1,38 +1,46 @@ -from reversion.admin import VersionAdmin - +# -*- coding: utf-8 -*- +""" +Logger admin module. +""" from django.contrib import admin -from onadata.apps.logger.models import XForm, Project +from reversion.admin import VersionAdmin + +from onadata.apps.logger.models import Project, XForm -class XFormAdmin(VersionAdmin, admin.ModelAdmin): - exclude = ('user',) - list_display = ('id_string', 'downloadable', 'shared') - search_fields = ('id_string', 'title') +class FilterByUserMixin: + """Filter queryset by ``request.user``.""" - # A user should only see forms that belong to him. + # A user should only see forms/projects that belong to him. def get_queryset(self, request): - qs = super(XFormAdmin, self).get_queryset(request) + """Returns queryset filtered by the `request.user`.""" + queryset = super().get_queryset(request) if request.user.is_superuser: - return qs - return qs.filter(user=request.user) + return queryset + return queryset.filter(**{self.user_lookup_field: request.user}) + + +class XFormAdmin(FilterByUserMixin, VersionAdmin, admin.ModelAdmin): + """Customise the XForm admin view.""" + + exclude = ("user",) + list_display = ("id_string", "downloadable", "shared") + search_fields = ("id_string", "title") + user_lookup_field = "user" admin.site.register(XForm, XFormAdmin) -class ProjectAdmin(VersionAdmin, admin.ModelAdmin): - list_max_show_all = 2000 - list_select_related = ('organization',) - ordering = ['name'] - search_fields = ('name', 'organization__username', 'organization__email') +class ProjectAdmin(FilterByUserMixin, VersionAdmin, admin.ModelAdmin): + """Customise the Project admin view.""" - # A user should only see projects that belong to him. - def get_queryset(self, request): - qs = super(ProjectAdmin, self).get_queryset(request) - if request.user.is_superuser: - return qs - return qs.filter(organization=request.user) + list_max_show_all = 2000 + list_select_related = ("organization",) + ordering = ["name"] + search_fields = ("name", "organization__username", "organization__email") + user_lookup_field = "organization" admin.site.register(Project, ProjectAdmin) diff --git a/onadata/apps/logger/factory.py b/onadata/apps/logger/factory.py index 40504df0db..7ce6529203 100644 --- a/onadata/apps/logger/factory.py +++ b/onadata/apps/logger/factory.py @@ -1,11 +1,15 @@ +# -*- coding: utf-8 -*- +""" +Factory utility functions. +""" # This factory is not the same as the others, and doesn't use # django-factories but it mimics their functionality... from datetime import timedelta -from pyxform import custom_values, Survey +from pyxform import Survey from pyxform.builder import create_survey_element_from_dict -from onadata.apps.logger.models import XForm, Instance +from onadata.apps.logger.models import Instance, XForm XFORM_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.000" ONE_HOUR = timedelta(0, 3600) @@ -16,22 +20,22 @@ def _load_registration_survey_object(): Loads a registration survey with all the values necessary to register a surveyor. """ - survey = Survey(name=u"registration", id_string=u"registration") - survey.add_child(create_survey_element_from_dict({ - u'type': u'text', u'name': u'name', u'label': u'Name' - })) - survey.add_child(create_survey_element_from_dict({ - u'type': u'start time', - u'name': u'start' - })) - survey.add_child(create_survey_element_from_dict({ - u'type': u'end time', - u'name': u'end' - })) - survey.add_child(create_survey_element_from_dict({ - u'type': u'imei', - u'name': u'device_id' - })) + survey = Survey(name="registration", id_string="registration") + survey.add_child( + create_survey_element_from_dict( + {"type": "text", "name": "name", "label": "Name"} + ) + ) + survey.add_child( + create_survey_element_from_dict({"type": "start time", "name": "start"}) + ) + survey.add_child( + create_survey_element_from_dict({"type": "end time", "name": "end"}) + ) + survey.add_child( + create_survey_element_from_dict({"type": "imei", "name": "device_id"}) + ) + return survey @@ -40,96 +44,109 @@ def _load_simple_survey_object(): Returns a "watersimple" survey object, complete with questions. """ - survey = Survey(name=u"WaterSimple", id_string=u"WaterSimple") - survey.add_child(create_survey_element_from_dict({ - u'hint': {u'English': u'What is this point named?'}, - u'label': {u'English': u'Water Point Name'}, - u'type': u'text', - u'name': u'name' - })) - survey.add_child(create_survey_element_from_dict({ - u'hint': {u'English': u'How many people use this every month?'}, - u'label': {u'English': u'Monthly Usage'}, - u'name': u'users_per_month', - u'type': u'integer' - })) - survey.add_child(create_survey_element_from_dict({ - u'type': u'gps', - u'name': u'geopoint', - u'label': {u'English': u'Location'} - })) - survey.add_child(create_survey_element_from_dict({ - u'type': u'imei', - u'name': u'device_id' - })) - survey.add_child(create_survey_element_from_dict({ - u'type': u'start time', - u'name': u'start' - })) - survey.add_child(create_survey_element_from_dict({ - u'type': u'end time', - u'name': u'end' - })) + survey = Survey(name="WaterSimple", id_string="WaterSimple") + survey.add_child( + create_survey_element_from_dict( + { + "hint": {"English": "What is this point named?"}, + "label": {"English": "Water Point Name"}, + "type": "text", + "name": "name", + } + ) + ) + survey.add_child( + create_survey_element_from_dict( + { + "hint": {"English": "How many people use this every month?"}, + "label": {"English": "Monthly Usage"}, + "name": "users_per_month", + "type": "integer", + } + ) + ) + survey.add_child( + create_survey_element_from_dict( + {"type": "gps", "name": "geopoint", "label": {"English": "Location"}} + ) + ) + survey.add_child( + create_survey_element_from_dict({"type": "imei", "name": "device_id"}) + ) + survey.add_child( + create_survey_element_from_dict({"type": "start time", "name": "start"}) + ) + survey.add_child( + create_survey_element_from_dict({"type": "end time", "name": "end"}) + ) + return survey -class XFormManagerFactory(object): +class XFormManagerFactory: + """XForm manager factory.""" def create_registration_xform(self): """ Calls 'get_registration_xform', saves the result, and returns. """ - xf = self.get_registration_xform() - xf.save() - return xf + xform = self.get_registration_xform() + xform.save() + return xform + + # pylint: disable=no-self-use def get_registration_xform(self): """ Gets a registration xform. (currently loaded in from fixture) Returns it without saving. """ reg_xform = _load_registration_survey_object() + return XForm(xml=reg_xform.to_xml()) - def create_registration_instance(self, custom_values={}): + def create_registration_instance(self, custom_values=None): + """Create registration instance.""" + custom_values = {} if custom_values is None else custom_values i = self.get_registration_instance(custom_values) i.save() return i - def get_registration_instance(self, custom_values={}): + def get_registration_instance(self, custom_values=None): """ 1. Checks to see if the registration form has been created alread. If not, it loads it in. 2. Loads a registration instance. """ - registration_xforms = XForm.objects.filter(title=u"registration") + custom_values = {} if custom_values is None else custom_values + registration_xforms = XForm.objects.filter(title="registration") if registration_xforms.count() == 0: - xf = self.create_registration_xform() + xform = self.create_registration_xform() else: - xf = registration_xforms[0] + xform = registration_xforms[0] values = { - u'device_id': u'12345', - u'start': u'2011-01-01T09:50:06.966', - u'end': u'2011-01-01T09:53:22.965' + "device_id": "12345", + "start": "2011-01-01T09:50:06.966", + "end": "2011-01-01T09:53:22.965", } - if u'start' in custom_values: - st = custom_values[u'start'] - custom_values[u'start'] = st.strftime(XFORM_TIME_FORMAT) + if "start" in custom_values: + start = custom_values["start"] + custom_values["start"] = start.strftime(XFORM_TIME_FORMAT) # if no end_time is specified, defaults to 1 hour - values[u'end'] = (st+ONE_HOUR).strftime(XFORM_TIME_FORMAT) + values["end"] = (start + ONE_HOUR).strftime(XFORM_TIME_FORMAT) - if u'end' in custom_values: - custom_values[u'end'] = custom_values[u'end'].strftime( - XFORM_TIME_FORMAT) + if "end" in custom_values: + custom_values["end"] = custom_values["end"].strftime(XFORM_TIME_FORMAT) values.update(custom_values) reg_xform = _load_registration_survey_object() reg_instance = reg_xform.instantiate() - reg_instance._id = xf.id_string + # pylint: disable=protected-access + reg_instance._id = xform.id_string for k, v in values.items(): reg_instance.answer(name=k, value=v) @@ -139,42 +156,43 @@ def get_registration_instance(self, custom_values={}): return Instance(xml=instance_xml) def create_simple_xform(self): - xf = self.get_simple_xform() - xf.save() - return xf + """Creates and returns xform.""" + xform = self.get_simple_xform() + xform.save() + + return xform def get_simple_xform(self): + """Returns a simple xform.""" survey_object = _load_simple_survey_object() return XForm(xml=survey_object.to_xml()) - i = self.get_simple_instance(custom_values) - i.save() - return i - def get_simple_instance(self, custom_values={}): - simple_xforms = XForm.objects.filter(title=u"WaterSimple") + def get_simple_instance(self, custom_values=None): + """Returns a simple submission instance.""" + custom_values = {} if custom_values is None else custom_values + simple_xforms = XForm.objects.filter(title="WaterSimple") if simple_xforms.count() == 0: - xf = self.create_simple_xform() + xform = self.create_simple_xform() else: - xf = simple_xforms[0] + xform = simple_xforms[0] # these values can be overridden with custom values values = { - u'device_id': u'12345', - u'start': u'2011-01-01T09:50:06.966', - u'end': u'2011-01-01T09:53:22.965', - u'geopoint': u'40.783594633609184 -73.96436698913574 300.0 4.0' + "device_id": "12345", + "start": "2011-01-01T09:50:06.966", + "end": "2011-01-01T09:53:22.965", + "geopoint": "40.783594633609184 -73.96436698913574 300.0 4.0", } - if u'start' in custom_values: - st = custom_values[u'start'] - custom_values[u'start'] = st.strftime(XFORM_TIME_FORMAT) + if "start" in custom_values: + start = custom_values["start"] + custom_values["start"] = start.strftime(XFORM_TIME_FORMAT) # if no end_time is specified, defaults to 1 hour - values[u'end'] = (st+ONE_HOUR).strftime(XFORM_TIME_FORMAT) + values["end"] = (start + ONE_HOUR).strftime(XFORM_TIME_FORMAT) - if u'end' in custom_values: - custom_values[u'end'] = custom_values[u'end'].strftime( - XFORM_TIME_FORMAT) + if "end" in custom_values: + custom_values["end"] = custom_values["end"].strftime(XFORM_TIME_FORMAT) values.update(custom_values) @@ -186,7 +204,8 @@ def get_simple_instance(self, custom_values={}): # setting the id_string so that it doesn't end up # with the timestamp of the new survey object - simple_survey._id = xf.id_string + # pylint: disable=protected-access + simple_survey._id = xform.id_string instance_xml = simple_survey.to_xml() diff --git a/onadata/apps/logger/import_tools.py b/onadata/apps/logger/import_tools.py index 4b0b017f95..f25d5b8a71 100644 --- a/onadata/apps/logger/import_tools.py +++ b/onadata/apps/logger/import_tools.py @@ -1,11 +1,15 @@ -# encoding=utf-8 +# -*- coding: utf-8 -*- +""" +Import forms and submission utility functions. +""" import os import shutil import tempfile import zipfile -from builtins import open +from contextlib import ExitStack from django.core.files.uploadedfile import InMemoryUploadedFile +from django.http.response import Http404 from onadata.apps.logger.xform_fs import XFormInstanceFS from onadata.celery import app @@ -30,90 +34,115 @@ def django_file(path, field_name, content_type): + """Returns an InMemoryUploadedFile object of a given file at the ``path``.""" # adapted from here: # http://groups.google.com/group/django-users/browse_thread/thread/ # 834f988876ff3c45/ - f = open(path, 'rb') + # pylint: disable=consider-using-with + f = open(path, "rb") return InMemoryUploadedFile( file=f, field_name=field_name, name=f.name, content_type=content_type, size=os.path.getsize(path), - charset=None + charset=None, ) -def import_instance(username, xform_path, photos, osm_files, status, - raise_exception): +# pylint: disable=too-many-arguments +def import_instance(username, xform_path, photos, osm_files, status): """ This callback is passed an instance of a XFormInstanceFS. See xform_fs.py for more info. """ - with django_file(xform_path, field_name="xml_file", - content_type="text/xml") as xml_file: - images = [django_file(jpg, field_name="image", - content_type="image/jpeg") for jpg in photos] - images += [ - django_file(osm, field_name='image', - content_type='text/xml') - for osm in osm_files + with ExitStack() as stack: + submission_file = stack.enter_context(open(xform_path, "rb")) + xml_file = stack.enter_context( + InMemoryUploadedFile( + submission_file, + "xml_file", + xform_path, + "text/xml", + os.path.getsize(xform_path), + None, + ) + ) + _media_files = [ + (stack.enter_context(open(path, "rb")), path, "image/jpeg") + for path in photos ] - try: - instance = create_instance(username, xml_file, images, status) - except Exception as e: - if raise_exception: - raise e - - for i in images: - i.close() + _media_files += [ + (stack.enter_context(open(path, "rb")), path, "text/xml") + for path in osm_files + ] + media_files = [ + stack.enter_context( + InMemoryUploadedFile( + open_file, + "image", + path, + content_type, + os.path.getsize(path), + None, + ) + ) + for open_file, path, content_type in _media_files + ] + instance = create_instance(username, xml_file, media_files, status) if instance: return 1 - else: - return 0 + return 0 @app.task(ignore_result=True) def import_instance_async(username, xform_path, photos, osm_files, status): - import_instance(username, xform_path, photos, osm_files, status, False) + """An async alias to import_instance() function.""" + import_instance(username, xform_path, photos, osm_files, status) -def iterate_through_instances(dirpath, callback, user=None, status='zip', - is_async=False): +def iterate_through_instances( + dirpath, callback, user=None, status="zip", is_async=False +): + """Iterate through all files and directories in the given ``dirpath``.""" total_file_count = 0 success_count = 0 errors = [] - for directory, subdirs, subfiles in os.walk(dirpath): + for directory, _subdirs, subfiles in os.walk(dirpath): for filename in subfiles: filepath = os.path.join(directory, filename) if XFormInstanceFS.is_valid_instance(filepath): xfxs = XFormInstanceFS(filepath) if is_async and user is not None: - callback.apply_async(( - user.username, xfxs.path, xfxs.photos, xfxs.osm, status - ), queue='instances') + callback.apply_async( + (user.username, xfxs.path, xfxs.photos, xfxs.osm, status), + queue="instances", + ) success_count += 1 else: try: - success_count += callback(xfxs) - except Exception as e: - errors.append("%s => %s" % (xfxs.filename, str(e))) - del(xfxs) + count = callback(xfxs) + except Http404: + pass + else: + if count: + success_count += count + del xfxs total_file_count += 1 return (total_file_count, success_count, errors) def import_instances_from_zip(zipfile_path, user, status="zip"): + """Unzips a zip file and imports submission instances from it.""" + temp_directory = tempfile.mkdtemp() try: - temp_directory = tempfile.mkdtemp() - zf = zipfile.ZipFile(zipfile_path) - - zf.extractall(temp_directory) + with zipfile.ZipFile(zipfile_path) as zip_file: + zip_file.extractall(temp_directory) except zipfile.BadZipfile as e: - errors = [u"%s" % e] + errors = [f"{e}"] return 0, 0, errors else: return import_instances_from_path(temp_directory, user, status) @@ -122,29 +151,22 @@ def import_instances_from_zip(zipfile_path, user, status="zip"): def import_instances_from_path(path, user, status="zip", is_async=False): + """Process all submission instances in the given directory tree at ``path``.""" + def callback(xform_fs): """ This callback is passed an instance of a XFormInstanceFS. See xform_fs.py for more info. """ - import_instance(user.username, - xform_fs.path, - xform_fs.photos, - xform_fs.osm, - status, - True) + import_instance( + user.username, xform_fs.path, xform_fs.photos, xform_fs.osm, status + ) if is_async: total_count, success_count, errors = iterate_through_instances( - path, - import_instance_async, - user=user, - status=status, - is_async=is_async + path, import_instance_async, user=user, status=status, is_async=is_async ) else: - total_count, success_count, errors = iterate_through_instances( - path, callback - ) + total_count, success_count, errors = iterate_through_instances(path, callback) return (total_count, success_count, errors) diff --git a/onadata/apps/logger/management/commands/add_id.py b/onadata/apps/logger/management/commands/add_id.py index f773354913..fe8651a7d8 100644 --- a/onadata/apps/logger/management/commands/add_id.py +++ b/onadata/apps/logger/management/commands/add_id.py @@ -1,5 +1,5 @@ from django.contrib.auth.models import User -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from django.core.management.base import BaseCommand from django.conf import settings @@ -8,8 +8,8 @@ class Command(BaseCommand): - args = '' - help = ugettext_lazy("Sync account with '_id'") + args = "" + help = gettext_lazy("Sync account with '_id'") def handle(self, *args, **kwargs): @@ -18,7 +18,7 @@ def handle(self, *args, **kwargs): users = User.objects.filter(username__contains=args[0]) else: # All the accounts - self.stdout.write("Fetching all the account {}", ending='\n') + self.stdout.write("Fetching all the account {}", ending="\n") users = User.objects.exclude( username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME ) @@ -27,24 +27,27 @@ def handle(self, *args, **kwargs): self.add_id(user) def add_id(self, user): - self.stdout.write("Syncing for account {}".format(user.username), - ending='\n') + self.stdout.write("Syncing for account {}".format(user.username), ending="\n") xforms = XForm.objects.filter(user=user) count = 0 failed = 0 - for instance in Instance.objects.filter( - xform__downloadable=True, xform__in=xforms)\ - .extra(where=['("logger_instance".json->>%s) is null'], - params=["_id"]).iterator(): + for instance in ( + Instance.objects.filter(xform__downloadable=True, xform__in=xforms) + .extra(where=['("logger_instance".json->>%s) is null'], params=["_id"]) + .iterator() + ): try: instance.save() count += 1 except Exception as e: failed += 1 - self.stdout.write(str(e), ending='\n') + self.stdout.write(str(e), ending="\n") pass - self.stdout.write("Syncing for account {}. Done. Success {}, Fail {}" - .format(user.username, count, failed), - ending='\n') + self.stdout.write( + "Syncing for account {}. Done. Success {}, Fail {}".format( + user.username, count, failed + ), + ending="\n", + ) diff --git a/onadata/apps/logger/management/commands/change_s3_media_permissions.py b/onadata/apps/logger/management/commands/change_s3_media_permissions.py index 63f8f23b1c..7fa04adc28 100644 --- a/onadata/apps/logger/management/commands/change_s3_media_permissions.py +++ b/onadata/apps/logger/management/commands/change_s3_media_permissions.py @@ -1,17 +1,24 @@ #!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=4 coding=utf-8 -import sys +# vim: ai ts=4 sts=4 et sw=4 +# -*- coding=utf-8 -*- +""" +change_s3_media_permissions - makes all s3 files private. +""" -from django.core.management.base import BaseCommand, CommandError from django.core.files.storage import get_storage_class -from django.utils.translation import ugettext as _, ugettext_lazy +from django.core.management.base import BaseCommand, CommandError +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy class Command(BaseCommand): - help = ugettext_lazy("Makes all s3 files private") + """Makes all s3 files private""" + + help = gettext_lazy("Makes all s3 files private") def handle(self, *args, **kwargs): - permissions = ('private', 'public-read', 'authenticated-read') + """Makes all s3 files private""" + permissions = ("private", "public-read", "authenticated-read") if len(args) < 1: raise CommandError(_("Missing permission argument")) @@ -19,24 +26,16 @@ def handle(self, *args, **kwargs): permission = args[0] if permission not in permissions: - raise CommandError(_( - "Expected %s as permission") % ' or '.join(permissions)) - - try: - s3 = get_storage_class('storages.backends.s3boto.S3BotoStorage')() - except Exception: - self.stderr.write(_( - u"Missing necessary libraries. Try running: pip install " - "-r requirements-s3.pip")) - sys.exit(1) - else: - all_files = s3.bucket.list() - - for i, f in enumerate(all_files): - f.set_acl(permission) - if i % 1000 == 0: - self.stdout.write(_( - "%s file objects processed" % i)) - - self.stdout.write(_( - "A total of %s file objects processed" % i)) + raise CommandError(_(f"Expected {' or '.join(permissions)} as permission")) + + s3_storage = get_storage_class("storages.backends.s3boto.S3BotoStorage")() + all_files = s3_storage.bucket.list() + + num = 0 + for i, f in enumerate(all_files): + f.set_acl(permission) + if i % 1000 == 0: + self.stdout.write(_(f"{i} file objects processed")) + num = i + + self.stdout.write(_(f"A total of {num} file objects processed")) diff --git a/onadata/apps/logger/management/commands/create_backup.py b/onadata/apps/logger/management/commands/create_backup.py index 6e95bd3bf0..9a05c5830f 100644 --- a/onadata/apps/logger/management/commands/create_backup.py +++ b/onadata/apps/logger/management/commands/create_backup.py @@ -1,37 +1,47 @@ +# -*- coding: utf-8 -*- +""" +create_backup - command to create zipped backup of a form and it's submissions. +""" import os +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import ugettext_lazy, ugettext as _ -from django.contrib.auth.models import User +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy from onadata.apps.logger.models import XForm from onadata.libs.utils.backup_tools import create_zip_backup class Command(BaseCommand): + """Create a zip backup of a form and all its submissions.""" + args = "outfile username [id_string]" - help = ugettext_lazy( - "Create a zip backup of a form and all its submissions") + help = gettext_lazy("Create a zip backup of a form and all its submissions") - def handle(self, *args, **options): + # pylint: disable=unused-argument + def handle(self, *args, **options): # noqa C901 + """Create a zip backup of a form and all its submissions.""" try: output_file = args[0] - except IndexError: - raise CommandError(_("Provide the path to the zip file to backup" - " to")) + except IndexError as e: + raise CommandError( + _("Provide the path to the zip file to backup to") + ) from e else: output_file = os.path.realpath(output_file) try: username = args[1] - except IndexError: - raise CommandError(_("You must provide the username to publish the" - " form to.")) + except IndexError as e: + raise CommandError( + _("You must provide the username to publish the form to.") + ) from e # make sure user exists try: - user = User.objects.get(username=username) - except User.DoesNotExist: - raise CommandError(_("The user '%s' does not exist.") % username) + user = get_user_model().objects.get(username=username) + except get_user_model().DoesNotExist as e: + raise CommandError(_(f"The user '{username}' does not exist.")) from e try: id_string = args[2] @@ -41,7 +51,8 @@ def handle(self, *args, **options): # make sure xform exists try: xform = XForm.objects.get(user=user, id_string=id_string) - except XForm.DoesNotExist: - raise CommandError(_("The id_string '%s' does not exist.") % - id_string) + except XForm.DoesNotExist as e: + raise CommandError( + _(f"The id_string '{id_string}' does not exist.") + ) from e create_zip_backup(output_file, user, xform) diff --git a/onadata/apps/logger/management/commands/create_image_thumbnails.py b/onadata/apps/logger/management/commands/create_image_thumbnails.py index a03718f33b..5c77b21d8b 100644 --- a/onadata/apps/logger/management/commands/create_image_thumbnails.py +++ b/onadata/apps/logger/management/commands/create_image_thumbnails.py @@ -1,10 +1,14 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +create_image_thumbnails - creates thumbnails for all form images and stores them. +""" from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.files.storage import get_storage_class from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy from onadata.apps.logger.models.attachment import Attachment from onadata.apps.logger.models.xform import XForm @@ -16,52 +20,55 @@ class Command(BaseCommand): - help = ugettext_lazy("Creates thumbnails for " - "all form images and stores them") + """Creates thumbnails for all form images and stores them""" + + help = gettext_lazy("Creates thumbnails for all form images and stores them") def add_arguments(self, parser): parser.add_argument( - '-u', - '--username', - help=ugettext_lazy("Username of the form user")) + "-u", "--username", help=gettext_lazy("Username of the form user") + ) parser.add_argument( - '-i', '--id_string', help=ugettext_lazy("id string of the form")) + "-i", "--id_string", help=gettext_lazy("id string of the form") + ) parser.add_argument( - '-f', - '--force', - action='store_false', - help=ugettext_lazy("regenerate thumbnails if they exist.")) + "-f", + "--force", + action="store_false", + help=gettext_lazy("regenerate thumbnails if they exist."), + ) + # pylint: disable=too-many-branches,too-many-locals def handle(self, *args, **options): attachments_qs = Attachment.objects.select_related( - 'instance', 'instance__xform') - if options.get('username'): - username = options.get('username') + "instance", "instance__xform" + ) + # pylint: disable=invalid-name + User = get_user_model() + if options.get("username"): + username = options.get("username") try: user = User.objects.get(username=username) - except User.DoesNotExist: - raise CommandError( - "Error: username %(username)s does not exist" % - {'username': username}) + except User.DoesNotExist as e: + raise CommandError(f"Error: username {username} does not exist") from e attachments_qs = attachments_qs.filter(instance__user=user) - if options.get('id_string'): - id_string = options.get('id_string') + if options.get("id_string"): + id_string = options.get("id_string") try: xform = XForm.objects.get(id_string=id_string) - except XForm.DoesNotExist: + except XForm.DoesNotExist as e: raise CommandError( - "Error: Form with id_string %(id_string)s does not exist" % - {'id_string': id_string}) + f"Error: Form with id_string {id_string} does not exist" + ) from e attachments_qs = attachments_qs.filter(instance__xform=xform) - fs = get_storage_class('django.core.files.storage.FileSystemStorage')() + fs = get_storage_class("django.core.files.storage.FileSystemStorage")() for att in queryset_iterator(attachments_qs): filename = att.media_file.name default_storage = get_storage_class()() - full_path = get_path(filename, - settings.THUMB_CONF['small']['suffix']) - if options.get('force') is not None: - for s in ['small', 'medium', 'large']: - fp = get_path(filename, settings.THUMB_CONF[s]['suffix']) + full_path = get_path(filename, settings.THUMB_CONF["small"]["suffix"]) + if options.get("force") is not None: + for s in ["small", "medium", "large"]: + fp = get_path(filename, settings.THUMB_CONF[s]["suffix"]) if default_storage.exists(fp): default_storage.delete(fp) if not default_storage.exists(full_path): @@ -70,17 +77,10 @@ def handle(self, *args, **options): resize(filename, att.extension) else: resize_local_env(filename, att.extension) - path = get_path( - filename, '%s' % THUMB_CONF['small']['suffix']) + path = get_path(filename, f'{THUMB_CONF["small"]["suffix"]}') if default_storage.exists(path): - self.stdout.write( - _(u'Thumbnails created for %(file)s') % - {'file': filename}) + self.stdout.write(_(f"Thumbnails created for {filename}")) else: - self.stdout.write( - _(u'Problem with the file %(file)s') % - {'file': filename}) - except (IOError, OSError) as e: - self.stderr.write(_( - u'Error on %(filename)s: %(error)s') - % {'filename': filename, 'error': e}) + self.stdout.write(_(f"Problem with the file {filename}")) + except (IOError, OSError) as error: + self.stderr.write(_(f"Error on {filename}: {error}")) diff --git a/onadata/apps/logger/management/commands/export_gps_points.py b/onadata/apps/logger/management/commands/export_gps_points.py index 15f5848b77..fec5259d0a 100644 --- a/onadata/apps/logger/management/commands/export_gps_points.py +++ b/onadata/apps/logger/management/commands/export_gps_points.py @@ -2,26 +2,26 @@ import csv from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models import Instance from onadata.libs.utils.model_tools import queryset_iterator class Command(BaseCommand): - help = ugettext_lazy("Export all gps points with their timestamps") + help = gettext_lazy("Export all gps points with their timestamps") def handle(self, *args, **kwargs): - with open('gps_points_export.csv', 'w') as csvfile: - fieldnames = ['longitude', 'latitude', 'date_created'] + with open("gps_points_export.csv", "w") as csvfile: + fieldnames = ["longitude", "latitude", "date_created"] writer = csv.writer(csvfile) writer.writerow(fieldnames) for instance in queryset_iterator( - Instance.objects.exclude(geom__isnull=True)): - if hasattr(instance, 'point') and instance.point is not None: + Instance.objects.exclude(geom__isnull=True) + ): + if hasattr(instance, "point") and instance.point is not None: longitude = instance.point.coords[0] latitude = instance.point.coords[1] - writer.writerow( - [longitude, latitude, instance.date_created]) + writer.writerow([longitude, latitude, instance.date_created]) self.stdout.write("Export of gps files has completed!!!!") diff --git a/onadata/apps/logger/management/commands/export_xforms_and_instances.py b/onadata/apps/logger/management/commands/export_xforms_and_instances.py index d983897cdb..28c7088473 100644 --- a/onadata/apps/logger/management/commands/export_xforms_and_instances.py +++ b/onadata/apps/logger/management/commands/export_xforms_and_instances.py @@ -1,10 +1,14 @@ #!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=4 coding=utf-8 +# vim: ai ts=4 sts=4 et sw=4 +# -*- coding=utf-8 -*- +""" +export_xformx_and_instances - exports XForms and submission instances into JSON files. +""" import os from django.core.management.base import BaseCommand from django.core.serializers import serialize -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from django.conf import settings from onadata.apps.logger.models import XForm, Instance @@ -12,9 +16,12 @@ class Command(BaseCommand): - help = ugettext_lazy("Export ODK forms and instances to JSON.") + """Export ODK forms and instances to JSON.""" + + help = gettext_lazy("Export ODK forms and instances to JSON.") def handle(self, *args, **kwargs): + """Export ODK forms and instances to JSON.""" fixtures_dir = os.path.join(PROJECT_ROOT, "json_xform_fixtures") if not os.path.exists(fixtures_dir): os.mkdir(fixtures_dir) @@ -22,10 +29,10 @@ def handle(self, *args, **kwargs): xform_fp = os.path.join(fixtures_dir, "a-xforms.json") instance_fp = os.path.join(fixtures_dir, "b-instances.json") - xfp = open(xform_fp, 'w') - xfp.write(serialize("json", XForm.objects.all())) - xfp.close() + with open(xform_fp, "w", encoding="utf-8") as xfp: + xfp.write(serialize("json", XForm.objects.all())) + xfp.close() - ifp = open(instance_fp, 'w') - ifp.write(serialize("json", Instance.objects.all())) - ifp.close() + with open(instance_fp, "w", encoding="utf-8") as ifp: + ifp.write(serialize("json", Instance.objects.all())) + ifp.close() diff --git a/onadata/apps/logger/management/commands/fix_attachments_counts.py b/onadata/apps/logger/management/commands/fix_attachments_counts.py index 00608cb913..edc2b203a6 100644 --- a/onadata/apps/logger/management/commands/fix_attachments_counts.py +++ b/onadata/apps/logger/management/commands/fix_attachments_counts.py @@ -6,8 +6,8 @@ from django.contrib.auth.models import User from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy from multidb.pinning import use_master from onadata.apps.logger.models.attachment import get_original_filename @@ -22,7 +22,8 @@ def update_attachments(instance): """ for attachment in instance.attachments.all(): attachment.name = os.path.basename( - get_original_filename(attachment.media_file.name)) + get_original_filename(attachment.media_file.name) + ) attachment.save() update_attachment_tracking(instance) @@ -31,18 +32,20 @@ class Command(BaseCommand): """ Fix attachments count command. """ - args = 'username' - help = ugettext_lazy("Fix attachments count.") + + args = "username" + help = gettext_lazy("Fix attachments count.") def add_arguments(self, parser): - parser.add_argument('username') + parser.add_argument("username") def handle(self, *args, **options): try: - username = options['username'] + username = options["username"] except KeyError: raise CommandError( - _("You must provide the username to publish the form to.")) + _("You must provide the username to publish the form to.") + ) # make sure user exists try: user = User.objects.get(username=username) @@ -56,16 +59,17 @@ def process_attachments(self, user): """ Process attachments for submissions where media_all_received is False. """ - xforms = XForm.objects.filter(user=user, deleted_at__isnull=True, - downloadable=True) + xforms = XForm.objects.filter( + user=user, deleted_at__isnull=True, downloadable=True + ) for xform in queryset_iterator(xforms): submissions = xform.instances.filter(media_all_received=False) to_process = submissions.count() if to_process: for submission in queryset_iterator(submissions): update_attachments(submission) - not_processed = xform.instances.filter( - media_all_received=False).count() - self.stdout.write("%s to process %s - %s = %s processed" % ( - xform, to_process, not_processed, - (to_process - not_processed))) + not_processed = xform.instances.filter(media_all_received=False).count() + self.stdout.write( + "%s to process %s - %s = %s processed" + % (xform, to_process, not_processed, (to_process - not_processed)) + ) diff --git a/onadata/apps/logger/management/commands/fix_duplicate_instances.py b/onadata/apps/logger/management/commands/fix_duplicate_instances.py index 618dfba306..2dfab73a58 100644 --- a/onadata/apps/logger/management/commands/fix_duplicate_instances.py +++ b/onadata/apps/logger/management/commands/fix_duplicate_instances.py @@ -3,13 +3,13 @@ from django.core.management.base import BaseCommand from django.db import connection -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models import Instance class Command(BaseCommand): - help = ugettext_lazy("Fix duplicate instances by merging the attachments.") + help = gettext_lazy("Fix duplicate instances by merging the attachments.") def query_data(self, sql): cursor = connection.cursor() @@ -18,14 +18,17 @@ def query_data(self, sql): yield row def handle(self, *args, **kwargs): - sql = "select xform_id, uuid, COUNT(xform_id || uuid) "\ - "from logger_instance group by xform_id, uuid "\ - "HAVING COUNT(xform_id || uuid) > 1;" + sql = ( + "select xform_id, uuid, COUNT(xform_id || uuid) " + "from logger_instance group by xform_id, uuid " + "HAVING COUNT(xform_id || uuid) > 1;" + ) total_count = 0 total_deleted = 0 for xform, uuid, dupes_count in self.query_data(sql): - instances = Instance.objects.filter(xform_id=xform, uuid=uuid)\ - .order_by('pk') + instances = Instance.objects.filter(xform_id=xform, uuid=uuid).order_by( + "pk" + ) first = instances[0] xml = instances[0].xml is_mspray_form = xform == 80970 @@ -44,9 +47,7 @@ def handle(self, *args, **kwargs): else: to_delete = instances.exclude(pk=first.pk) - media_files = list( - first.attachments.values_list('media_file', flat=True) - ) + media_files = list(first.attachments.values_list("media_file", flat=True)) delete_count = 0 for i in to_delete: delete_count += 1 @@ -58,12 +59,13 @@ def handle(self, *args, **kwargs): if delete_count >= dupes_count: raise AssertionError( "# of records to delete %d should be less than total # of " - "duplicates %d." % (delete_count, dupes_count)) + "duplicates %d." % (delete_count, dupes_count) + ) to_delete.delete() total_count += dupes_count total_deleted += delete_count - self.stdout.write("deleted %d: %s (%d of %d)." - % (xform, uuid, delete_count, dupes_count)) + self.stdout.write( + "deleted %d: %s (%d of %d)." % (xform, uuid, delete_count, dupes_count) + ) - self.stdout.write("done: deleted %d of %d" - % (total_deleted, total_count)) + self.stdout.write("done: deleted %d of %d" % (total_deleted, total_count)) diff --git a/onadata/apps/logger/management/commands/fix_submission_count.py b/onadata/apps/logger/management/commands/fix_submission_count.py index 1f920239a9..5cc99a0c17 100644 --- a/onadata/apps/logger/management/commands/fix_submission_count.py +++ b/onadata/apps/logger/management/commands/fix_submission_count.py @@ -3,7 +3,7 @@ from django.core.management.base import BaseCommand from django.db import transaction -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models import Instance from onadata.apps.logger.models import XForm @@ -11,32 +11,35 @@ class Command(BaseCommand): - help = ugettext_lazy("Fix num of submissions") + help = gettext_lazy("Fix num of submissions") def handle(self, *args, **kwargs): i = 0 xform_count = XForm.objects.filter(downloadable=True).count() for xform in XForm.objects.filter(downloadable=True).iterator(): with transaction.atomic(): - instance_count = xform.instances.filter(deleted_at=None)\ - .count() + instance_count = xform.instances.filter(deleted_at=None).count() xform.num_of_submissions = instance_count - xform.save(update_fields=['num_of_submissions']) + xform.save(update_fields=["num_of_submissions"]) i += 1 - self.stdout.write('Processing {} of {}: {} ({})'.format( - i, xform_count, xform.id_string, instance_count)) + self.stdout.write( + "Processing {} of {}: {} ({})".format( + i, xform_count, xform.id_string, instance_count + ) + ) i = 0 profile_count = UserProfile.objects.count() - for profile in UserProfile.objects.select_related('user__username')\ - .iterator(): + for profile in UserProfile.objects.select_related("user__username").iterator(): with transaction.atomic(): instance_count = Instance.objects.filter( - deleted_at=None, - xform__user_id=profile.user_id + deleted_at=None, xform__user_id=profile.user_id ).count() profile.num_of_submissions = instance_count - profile.save(update_fields=['num_of_submissions']) + profile.save(update_fields=["num_of_submissions"]) i += 1 - self.stdout.write('Processing {} of {}: {} ({})'.format( - i, profile_count, profile.user.username, instance_count)) + self.stdout.write( + "Processing {} of {}: {} ({})".format( + i, profile_count, profile.user.username, instance_count + ) + ) diff --git a/onadata/apps/logger/management/commands/generate_platform_stats.py b/onadata/apps/logger/management/commands/generate_platform_stats.py index 093377ad08..b550cc10de 100644 --- a/onadata/apps/logger/management/commands/generate_platform_stats.py +++ b/onadata/apps/logger/management/commands/generate_platform_stats.py @@ -16,8 +16,7 @@ def _write_stats_to_file(month: int, year: int): - out_file = open( - f"/tmp/platform_statistics_{month}_{year}.csv", "w") # nosec + out_file = open(f"/tmp/platform_statistics_{month}_{year}.csv", "w") # nosec writer = csv.writer(out_file) headers = ["Username", "Project Name", "Form Title", "No. of submissions"] writer.writerow(headers) @@ -26,17 +25,23 @@ def _write_stats_to_file(month: int, year: int): forms = XForm.objects.filter( Q(deleted_at__isnull=True) | Q(deleted_at__gt=date_obj), - date_created__lte=date_obj - ).values('id', 'project__name', 'project__organization__username', 'title') + date_created__lte=date_obj, + ).values("id", "project__name", "project__organization__username", "title") with use_master: for form in forms: instance_count = Instance.objects.filter( Q(deleted_at__isnull=True) | Q(deleted_at__gt=date_obj), - xform_id=form.get('id'), date_created__lte=date_obj + xform_id=form.get("id"), + date_created__lte=date_obj, ).count() writer.writerow( - [form.get('project__organization__username'), - form.get('project__name'), form.get('title'), instance_count]) + [ + form.get("project__organization__username"), + form.get("project__name"), + form.get("title"), + instance_count, + ] + ) class Command(BaseCommand): @@ -45,27 +50,30 @@ class Command(BaseCommand): information about the number of organizations, users, projects & submissions """ + help = _("Generates system statistics for the entire platform") def add_arguments(self, parser): parser.add_argument( - '--month', - '-m', - dest='month', - help=('Month to calculate system statistics for.' - 'Defaults to current month.'), - default=None + "--month", + "-m", + dest="month", + help=( + "Month to calculate system statistics for." "Defaults to current month." + ), + default=None, ) parser.add_argument( - '--year', - '-y', - dest='year', - help=('Year to calculate system statistics for.' - ' Defaults to current year'), + "--year", + "-y", + dest="year", + help=( + "Year to calculate system statistics for." " Defaults to current year" + ), default=None, ) def handle(self, *args, **options): - month = int(options.get('month', datetime.now().month)) - year = int(options.get('year', datetime.now().year)) + month = int(options.get("month", datetime.now().month)) + year = int(options.get("year", datetime.now().year)) _write_stats_to_file(month, year) diff --git a/onadata/apps/logger/management/commands/import.py b/onadata/apps/logger/management/commands/import.py index 7c6dc3a25d..f557c1788c 100644 --- a/onadata/apps/logger/management/commands/import.py +++ b/onadata/apps/logger/management/commands/import.py @@ -4,13 +4,13 @@ import os from django.core.management.base import BaseCommand from django.core.management import call_command -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy class Command(BaseCommand): - help = ugettext_lazy("Import ODK forms and instances.") + help = gettext_lazy("Import ODK forms and instances.") def handle(self, *args, **kwargs): path = args[0] - call_command('import_forms', os.path.join(path, "forms")) - call_command('import_instances', os.path.join(path, "instances")) + call_command("import_forms", os.path.join(path, "forms")) + call_command("import_instances", os.path.join(path, "instances")) diff --git a/onadata/apps/logger/management/commands/import_briefcase.py b/onadata/apps/logger/management/commands/import_briefcase.py index 57ba797d8f..09c238a727 100644 --- a/onadata/apps/logger/management/commands/import_briefcase.py +++ b/onadata/apps/logger/management/commands/import_briefcase.py @@ -1,28 +1,37 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +import_briefcase command -from django.contrib.auth.models import User +- imports XForm XML from a folder and publishes the XForm. +- import XForm submissions XML from a folder and inserts into the table instances. +""" + +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from onadata.libs.utils.briefcase_client import BriefcaseClient class Command(BaseCommand): + """Insert all existing parsed instances into MongoDB""" + help = _("Insert all existing parsed instances into MongoDB") def add_arguments(self, parser): - parser.add_argument( - '--url', help=_("server url to pull forms and submissions")) - parser.add_argument('-u', '--username', help=_("Username")), - parser.add_argument('-p', '--password', help=_("Password")) - parser.add_argument('--to', help=_("username in this server")) + parser.add_argument("--url", help=_("server url to pull forms and submissions")) + parser.add_argument("-u", "--username", help=_("Username")) + parser.add_argument("-p", "--password", help=_("Password")) + parser.add_argument("--to", help=_("username in this server")) def handle(self, *args, **options): - url = options.get('url') - username = options.get('username') - password = options.get('password') - to = options.get('to') - user = User.objects.get(username=to) - bc = BriefcaseClient( - username=username, password=password, user=user, url=url) - bc.push() + """Insert all existing parsed instances into MongoDB""" + url = options.get("url") + username = options.get("username") + password = options.get("password") + user = get_user_model().objects.get(username=options.get("to")) + client = BriefcaseClient( + username=username, password=password, user=user, url=url + ) + client.push() diff --git a/onadata/apps/logger/management/commands/import_forms.py b/onadata/apps/logger/management/commands/import_forms.py index 7c0c0f59a7..f33a592042 100644 --- a/onadata/apps/logger/management/commands/import_forms.py +++ b/onadata/apps/logger/management/commands/import_forms.py @@ -1,22 +1,29 @@ #!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=4 coding=utf-8 +# -*- coding: utf-8 -*- +# vim: ai ts=4 sts=4 et sw=4 +""" +import_forms - loads XForms from a given path. +""" from __future__ import absolute_import import glob import os from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models import XForm class Command(BaseCommand): - help = ugettext_lazy("Import a folder of XForms for ODK.") + """Import a folder of XForms for ODK.""" + help = gettext_lazy("Import a folder of XForms for ODK.") + + # pylint: disable=unused-argument def handle(self, *args, **kwargs): + """Import a folder of XForms for ODK.""" path = args[0] for form in glob.glob(os.path.join(path, "*")): - f = open(form) - XForm.objects.get_or_create(xml=f.read(), downloadable=False) - f.close() + with open(form, encoding="utf-8") as f: + XForm.objects.get_or_create(xml=f.read(), downloadable=False) diff --git a/onadata/apps/logger/management/commands/import_instances.py b/onadata/apps/logger/management/commands/import_instances.py index eab0d33195..b54a364025 100644 --- a/onadata/apps/logger/management/commands/import_instances.py +++ b/onadata/apps/logger/management/commands/import_instances.py @@ -1,65 +1,84 @@ #!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=5 coding=utf-8 +# vim: ai ts=4 sts=4 et sw=5 +# -*- coding: utf-8 -*- +""" +import_instances - import ODK instances from a zipped file. +""" import os -from past.builtins import basestring -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy -from onadata.apps.logger.import_tools import (import_instances_from_path, - import_instances_from_zip) +from onadata.apps.logger.import_tools import ( + import_instances_from_path, + import_instances_from_zip, +) + +# pylint: disable=invalid-name +User = get_user_model() class Command(BaseCommand): - args = 'username path' - help = ugettext_lazy("Import a zip file, a directory containing zip files " - "or a directory of ODK instances") + """ + import_instances - import ODK instances from a zipped file. + """ + + args = "username path" + help = gettext_lazy( + "Import a zip file, a directory containing zip files " + "or a directory of ODK instances" + ) def _log_import(self, results): total_count, success_count, errors = results - self.stdout.write(_( - "Total: %(total)d, Imported: %(imported)d, Errors: " - "%(errors)s\n------------------------------\n") % { - 'total': total_count, 'imported': success_count, - 'errors': errors}) + self.stdout.write( + _( + "Total: %(total)d, Imported: %(imported)d, Errors: " + "%(errors)s\n------------------------------\n" + ) + % {"total": total_count, "imported": success_count, "errors": errors} + ) + # pylint: disable=unused-argument def handle(self, *args, **kwargs): if len(args) < 2: raise CommandError(_("Usage: username file/path.")) username = args[0] path = args[1] - is_async = args[2] if len(args) > 2 else False - is_async = True if isinstance(is_async, basestring) and \ - is_async.lower() == 'true' else False + is_async = False + if len(args) > 2: + if isinstance(args[2], str): + is_async = args[2].lower() == "true" + try: user = User.objects.get(username=username) - except User.DoesNotExist: - raise CommandError(_( - "The specified user '%s' does not exist.") % username) + except User.DoesNotExist as e: + raise CommandError( + _(f"The specified user '{username}' does not exist.") + ) from e # make sure path exists if not os.path.exists(path): - raise CommandError(_( - "The specified path '%s' does not exist.") % path) + raise CommandError(_(f"The specified path '{path}' does not exist.")) - for dir, subdirs, files in os.walk(path): - # check if the dir has an odk directory + for directory, subdirs, files in os.walk(path): + # check if the directory has an odk directory if "odk" in subdirs: - # dont walk further down this dir + # dont walk further down this directory subdirs.remove("odk") - self.stdout.write(_("Importing from dir %s..\n") % dir) - results = import_instances_from_path( - dir, user, is_async=is_async - ) + self.stdout.write(_(f"Importing from directory {directory}..\n")) + results = import_instances_from_path(directory, user, is_async=is_async) self._log_import(results) for file in files: filepath = os.path.join(path, file) - if os.path.isfile(filepath) and\ - os.path.splitext(filepath)[1].lower() == ".zip": - self.stdout.write(_( - "Importing from zip at %s..\n") % filepath) + is_zip_file = ( + os.path.isfile(filepath) + and os.path.splitext(filepath)[1].lower() == ".zip" + ) + if is_zip_file: + self.stdout.write(_(f"Importing from zip at {filepath}..\n")) results = import_instances_from_zip(filepath, user) self._log_import(results) diff --git a/onadata/apps/logger/management/commands/import_tools.py b/onadata/apps/logger/management/commands/import_tools.py index 9383e143ef..fbb8eb7d35 100644 --- a/onadata/apps/logger/management/commands/import_tools.py +++ b/onadata/apps/logger/management/commands/import_tools.py @@ -1,46 +1,49 @@ #!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=4 coding=utf-8 - +# vim: ai ts=4 sts=4 et sw=4 +# -*- coding=utf-8 -*- +""" +import_tools - import ODK formms and instances. +""" import glob import os from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import ugettext as _, ugettext_lazy - -from onadata.libs.logger.import_tools import import_instances_from_zip -from onadata.libs.logger.models import Instance +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy +from onadata.apps.logger.import_tools import import_instances_from_zip +from onadata.apps.logger.models import Instance IMAGES_DIR = os.path.join(settings.MEDIA_ROOT, "attachments") class Command(BaseCommand): - help = ugettext_lazy("Import ODK forms and instances.") + """Import ODK forms and instances.""" + + help = gettext_lazy("Import ODK forms and instances.") def handle(self, *args, **kwargs): + """Import ODK forms and instances.""" if args.__len__() < 2: - raise CommandError(_(u"path(xform instances) username")) + raise CommandError(_("path(xform instances) username")) path = args[0] username = args[1] try: - user = User.objects.get(username=username) - except User.DoesNotExist: - raise CommandError(_(u"Invalid username %s") % username) + user = get_user_model().objects.get(username=username) + except get_user_model().DoesNotExist as e: + raise CommandError(_(f"Invalid username {username}")) from e debug = False if debug: - self.stdout.write(_(u"[Importing XForm Instances from %(path)s]\n") - % {'path': path}) - im_count = len(glob.glob(os.path.join(IMAGES_DIR, '*'))) - self.stdout.write(_(u"Before Parse:")) - self.stdout.write(_(u" --> Images: %(nb)d") % {'nb': im_count}) - self.stdout.write((_(u" --> Instances: %(nb)d") - % {'nb': Instance.objects.count()})) + self.stdout.write(_(f"[Importing XForm Instances from {path}]\n")) + im_count = len(glob.glob(os.path.join(IMAGES_DIR, "*"))) + self.stdout.write(_("Before Parse:")) + self.stdout.write(_(f" --> Images: {im_count}")) + self.stdout.write((_(f" --> Instances: {Instance.objects.count()}"))) import_instances_from_zip(path, user) if debug: - im_count2 = len(glob.glob(os.path.join(IMAGES_DIR, '*'))) - self.stdout.write(_(u"After Parse:")) - self.stdout.write(_(u" --> Images: %(nb)d") % {'nb': im_count2}) - self.stdout.write((_(u" --> Instances: %(nb)d") - % {'nb': Instance.objects.count()})) + im_count2 = len(glob.glob(os.path.join(IMAGES_DIR, "*"))) + self.stdout.write(_("After Parse:")) + self.stdout.write(_(f" --> Images: {im_count2}")) + self.stdout.write((_(f" --> Instances: {Instance.objects.count()}"))) diff --git a/onadata/apps/logger/management/commands/move_media_to_s3.py b/onadata/apps/logger/management/commands/move_media_to_s3.py index f2d8fbc213..a123cc1e5d 100644 --- a/onadata/apps/logger/management/commands/move_media_to_s3.py +++ b/onadata/apps/logger/management/commands/move_media_to_s3.py @@ -1,59 +1,67 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +move_media_to_s3 - Moves all XLSForm file from local storage to S3 storage. +""" import sys from django.core.files.storage import get_storage_class from django.core.management.base import BaseCommand -from django.utils.translation import ugettext as _, ugettext_lazy +from django.utils.translation import gettext as _, gettext_lazy from onadata.apps.logger.models.attachment import Attachment -from onadata.apps.logger.models.attachment import upload_to as\ - attachment_upload_to -from onadata.apps.logger.models.xform import XForm, upload_to as\ - xform_upload_to +from onadata.apps.logger.models.attachment import upload_to as attachment_upload_to +from onadata.apps.logger.models.xform import XForm, upload_to as xform_upload_to class Command(BaseCommand): - help = ugettext_lazy("Moves all attachments and xls files " - "to s3 from the local file system storage.") + """Moves all XLSForm file from local storage to S3 storage.""" + help = gettext_lazy( + "Moves all attachments and xls files " + "to s3 from the local file system storage." + ) + + # pylint: disable=unused-argument def handle(self, *args, **kwargs): - try: - fs = get_storage_class( - 'django.core.files.storage.FileSystemStorage')() - s3 = get_storage_class('storages.backends.s3boto.S3BotoStorage')() - except Exception: - self.stderr.write(_( - u"Missing necessary libraries. Try running: pip install -r" - "requirements/s3.pip")) - sys.exit(1) + """Moves all XLSForm file from local storage to S3 storage.""" + local_fs = get_storage_class("django.core.files.storage.FileSystemStorage")() + s3_fs = get_storage_class("storages.backends.s3boto.S3BotoStorage")() default_storage = get_storage_class()() - if default_storage.__class__ != s3.__class__: - self.stderr.write(_( - u"You must first set your default storage to s3 in your " - "local_settings.py file.")) + if default_storage.__class__ != s3_fs.__class__: + self.stderr.write( + _( + "You must first set your default storage to s3 in your " + "local_settings.py file." + ) + ) sys.exit(1) classes_to_move = [ - (Attachment, 'media_file', attachment_upload_to), - (XForm, 'xls', xform_upload_to), + (Attachment, "media_file", attachment_upload_to), + (XForm, "xls", xform_upload_to), ] for cls, file_field, upload_to in classes_to_move: - self.stdout.write(_( - u"Moving %(class)ss to s3...") % {'class': cls.__name__}) + self.stdout.write(_("Moving %(class)ss to s3...") % {"class": cls.__name__}) for i in cls.objects.all(): f = getattr(i, file_field) old_filename = f.name - if f.name and fs.exists(f.name) and not s3.exists( - upload_to(i, f.name)): - f.save(fs.path(f.name), fs.open(fs.path(f.name))) - self.stdout.write(_( - "\t+ '%(fname)s'\n\t---> '%(url)s'") - % {'fname': fs.path(old_filename), 'url': f.url}) + if ( + f.name + and local_fs.exists(f.name) + and not s3_fs.exists(upload_to(i, f.name)) + ): + f.save(local_fs.path(f.name), local_fs.open(local_fs.path(f.name))) + self.stdout.write( + _("\t+ '%(fname)s'\n\t---> '%(url)s'") + % {"fname": local_fs.path(old_filename), "url": f.url} + ) else: + exists_locally = local_fs.exists(f.name) + exists_s3 = not s3_fs.exists(upload_to(i, f.name)) self.stderr.write( - "\t- (f.name=%s, fs.exists(f.name)=%s, not s3.exist" - "s(upload_to(i, f.name))=%s)" % ( - f.name, fs.exists(f.name), - not s3.exists(upload_to(i, f.name)))) + f"\t- (f.name={f.name}, fs.exists(f.name)={exists_locally}," + f" not s3.exist s3upload_to(i, f.name))={exists_s3})" + ) diff --git a/onadata/apps/logger/management/commands/populate_osmdata_model.py b/onadata/apps/logger/management/commands/populate_osmdata_model.py index 58603cc276..867cbdd030 100644 --- a/onadata/apps/logger/management/commands/populate_osmdata_model.py +++ b/onadata/apps/logger/management/commands/populate_osmdata_model.py @@ -1,4 +1,8 @@ -from django.utils.translation import ugettext_lazy +# -*- coding: utf-8 -*- +""" +populate_osmdata_model command - process OSM XML and save data in OsmData model/table. +""" +from django.utils.translation import gettext_lazy from django.core.management.base import BaseCommand from onadata.apps.logger.models import Attachment @@ -8,10 +12,14 @@ class Command(BaseCommand): - args = '' - help = ugettext_lazy("Populate OsmData model with osm info.") + """Populate OsmData model with osm info.""" + args = "" + help = gettext_lazy("Populate OsmData model with osm info.") + + # pylint: disable=unused-argument def handle(self, *args, **kwargs): + """Populate OsmData model with osm info.""" xforms = XForm.objects.filter(instances_with_osm=True) # username @@ -20,20 +28,22 @@ def handle(self, *args, **kwargs): for xform in queryset_iterator(xforms): attachments = Attachment.objects.filter( - extension=Attachment.OSM, - instance__xform=xform - ).distinct('instance') + extension=Attachment.OSM, instance__xform=xform + ).distinct("instance") count = attachments.count() - c = 0 - for a in queryset_iterator(attachments): - pk = a.instance.parsed_instance.pk - save_osm_data(pk) - c += 1 - if c % 1000 == 0: - self.stdout.write("%s:%s: Processed %s of %s." % - (xform.user.username, - xform.id_string, c, count)) - - self.stdout.write("%s:%s: processed %s of %s submissions." % - (xform.user.username, xform.id_string, c, count)) + counter = 0 + for attachment in queryset_iterator(attachments): + instance_pk = attachment.instance.parsed_instance.pk + save_osm_data(instance_pk) + counter += 1 + if counter % 1000 == 0: + self.stdout.write( + f"{xform.user.username}:{xform.id_string}: " + f"Processed {counter} of {count}." + ) + + self.stdout.write( + f"{xform.user.username}:{xform.id_string}: " + f"processed {counter} of {count} submissions." + ) diff --git a/onadata/apps/logger/management/commands/publish_xls.py b/onadata/apps/logger/management/commands/publish_xls.py index d04a13e3a6..bd5f53851d 100644 --- a/onadata/apps/logger/management/commands/publish_xls.py +++ b/onadata/apps/logger/management/commands/publish_xls.py @@ -1,84 +1,107 @@ +# -*- coding: utf-8 -*- +""" +publish_xls - Publish an XLSForm command. +""" import os -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model +from django.core.files.uploadedfile import InMemoryUploadedFile from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy + from pyxform.builder import create_survey_from_xls from onadata.apps.logger.models.project import Project from onadata.apps.logger.models.xform import XForm from onadata.libs.utils.logger_tools import publish_xls_form from onadata.libs.utils.user_auth import get_user_default_project -from onadata.libs.utils.viewer_tools import django_file class Command(BaseCommand): - args = 'xls_file username project' - help = ugettext_lazy("Publish an XLS file with the option of replacing an" - "existing one") + """Publish an XLSForm file.""" + + args = "xls_file username project" + help = gettext_lazy( + "Publish an XLS file with the option of replacing an" "existing one" + ) def add_arguments(self, parser): - parser.add_argument('xls_filepath') - parser.add_argument('username') + parser.add_argument("xls_filepath") + parser.add_argument("username") parser.add_argument( - '-p', '--project-name', action='store_true', dest='project_name') + "-p", "--project-name", action="store_true", dest="project_name" + ) parser.add_argument( - '-r', - '--replace', - action='store_true', - dest='replace', - help=ugettext_lazy("Replace existing form if any")) + "-r", + "--replace", + action="store_true", + dest="replace", + help=gettext_lazy("Replace existing form if any"), + ) - def handle(self, *args, **options): + def handle(self, *args, **options): # noqa C901 + # pylint: disable=invalid-name + User = get_user_model() # noqa N806 try: - xls_filepath = options['xls_filepath'] - except KeyError: - raise CommandError(_("You must provide the path to the xls file.")) + xls_filepath = options["xls_filepath"] + except KeyError as e: + raise CommandError(_("You must provide the path to the xls file.")) from e # make sure path exists if not os.path.exists(xls_filepath): - raise CommandError( - _("The xls file '%s' does not exist.") % xls_filepath) + raise CommandError(_(f"The xls file '{xls_filepath}' does not exist.")) try: - username = options['username'] - except KeyError: + username = options["username"] + except KeyError as e: raise CommandError( - _("You must provide the username to publish the form to.")) + _("You must provide the username to publish the form to.") + ) from e # make sure user exists try: user = User.objects.get(username=username) - except User.DoesNotExist: - raise CommandError(_("The user '%s' does not exist.") % username) + except User.DoesNotExist as e: + raise CommandError(_(f"The user '{username}' does not exist.")) from e # wasteful but we need to get the id_string beforehand survey = create_survey_from_xls(xls_filepath) # check if a form with this id_string exists for this user - form_already_exists = XForm.objects.filter( - user=user, id_string=survey.id_string).count() > 0 + form_already_exists = ( + XForm.objects.filter(user=user, id_string=survey.id_string).count() > 0 + ) # id_string of form to replace, if any id_string = None if form_already_exists: - if 'replace' in options and options['replace']: + if "replace" in options and options["replace"]: id_string = survey.id_string self.stdout.write(_("Form already exist, replacing ..\n")) else: raise CommandError( - _("The form with id_string '%s' already exists, use the -r" - " option to replace it.") % survey.id_string) + _( + f"The form with id_string '{survey.id_string}' already exists," + " use the -r option to replace it." + ) + ) else: self.stdout.write(_("Form does NOT exist, publishing ..\n")) try: - project_name = options['project_name'] + project_name = options["project_name"] project = Project.objects.get(name=project_name) except (KeyError, Project.DoesNotExist): project = get_user_default_project(user) # publish - xls_file = django_file(xls_filepath, 'xls_file', - 'application/vnd.ms-excel') - publish_xls_form(xls_file, user, project, id_string) - self.stdout.write(_("Done..\n")) + with open(xls_filepath, "rb") as file_object: + with InMemoryUploadedFile( + file=file_object, + field_name="xls_file", + name=file_object.name, + content_type="application/vnd.ms-excel", + size=os.path.getsize(xls_filepath), + charset=None, + ) as xls_file: + publish_xls_form(xls_file, user, project, id_string) + self.stdout.write(_("Done..\n")) diff --git a/onadata/apps/logger/management/commands/pull_from_aggregate.py b/onadata/apps/logger/management/commands/pull_from_aggregate.py index cbb0bdaea5..07b7f75384 100644 --- a/onadata/apps/logger/management/commands/pull_from_aggregate.py +++ b/onadata/apps/logger/management/commands/pull_from_aggregate.py @@ -1,33 +1,46 @@ #!/usr/bin/env python +# -*- coding=utf-8 -*- +""" +pull_from_aggregate command -from django.contrib.auth.models import User +Uses the BriefcaseClient to download forms and submissions +from a server that implements the Briefcase Aggregate API. +""" + +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from onadata.libs.utils.briefcase_client import BriefcaseClient class Command(BaseCommand): - help = _("Insert all existing parsed instances into MongoDB") + """Download forms and submissions from a server with Briefcase Aggregate API""" + + help = _( + "Download forms and submissions from a server with Briefcase Aggregate API" + ) def add_arguments(self, parser): - parser.add_argument( - '--url', help=_("server url to pull forms and submissions")) - parser.add_argument('-u', '--username', help=_("Username")) - parser.add_argument('-p', '--password', help=_("Password")) - parser.add_argument('--to', help=_("username in this server")) + """Download forms and submissions from a server with Briefcase Aggregate API""" + parser.add_argument("--url", help=_("server url to pull forms and submissions")) + parser.add_argument("-u", "--username", help=_("Username")) + parser.add_argument("-p", "--password", help=_("Password")) + parser.add_argument("--to", help=_("username in this server")) def handle(self, *args, **kwargs): - url = kwargs.get('url') - username = kwargs.get('username') - password = kwargs.get('password') - to = kwargs.get('to') - if username is None or password is None or to is None or url is None: + url = kwargs.get("url") + username = kwargs.get("username") + password = kwargs.get("password") + to_username = kwargs.get("to") + if username is None or password is None or to_username is None or url is None: self.stderr.write( - 'pull_form_aggregate -u username -p password --to=username' - ' --url=aggregate_server_url') + "pull_from_aggregate -u username -p password --to=username" + " --url=aggregate_server_url" + ) else: - user = User.objects.get(username=to) - bc = BriefcaseClient( - username=username, password=password, user=user, url=url) - bc.download_xforms(include_instances=True) + user = get_user_model().objects.get(username=to_username) + briefcase_client = BriefcaseClient( + username=username, password=password, user=user, url=url + ) + briefcase_client.download_xforms(include_instances=True) diff --git a/onadata/apps/logger/management/commands/reapplyperms.py b/onadata/apps/logger/management/commands/reapplyperms.py index 20060f81f6..a38764ee70 100644 --- a/onadata/apps/logger/management/commands/reapplyperms.py +++ b/onadata/apps/logger/management/commands/reapplyperms.py @@ -1,23 +1,29 @@ #!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=4 coding=utf-8 +# -*- coding: utf-8 -*- +# vim: ai ts=4 sts=4 et sw=4 +""" +reapplyperms - reapplies XForm permissions for the given user. +""" import gc - -from django.utils import timezone from datetime import timedelta -from onadata.apps.logger.models import XForm -from django.db.models import Q + from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy as _ +from django.db.models import Q +from django.utils import timezone +from django.utils.translation import gettext_lazy as _ from multidb.pinning import use_master -from onadata.libs.utils.project_utils import set_project_perms_to_xform +from onadata.apps.logger.models import XForm from onadata.libs.utils.model_tools import queryset_iterator +from onadata.libs.utils.project_utils import set_project_perms_to_xform DEFAULT_SECONDS = 30 * 60 class Command(BaseCommand): + """Reapply permissions to XForms.""" + help = _("Reapply permissions to XForms.") def _reapply_perms(self, username=None, days=None, seconds=None): @@ -35,31 +41,37 @@ def _reapply_perms(self, username=None, days=None, seconds=None): if the_past: xforms = XForm.objects.filter( - Q(date_created__gte=the_past) | - Q(date_modified__gte=the_past)) + Q(date_created__gte=the_past) | Q(date_modified__gte=the_past) + ) - self.stdout.write(_("{} to be updated").format(xforms.count())) + self.stdout.write(_(f"{xforms.count()} to be updated")) with use_master: for xform in queryset_iterator(xforms): set_project_perms_to_xform(xform, xform.project) - self.stdout.write( - _(f"Updated permissions for XForm ID: {xform.id}")) + self.stdout.write(_(f"Updated permissions for XForm ID: {xform.id}")) gc.collect() def add_arguments(self, parser): - parser.add_argument('--days', dest='days', type=int, default=0, - help=_("No of days")) - parser.add_argument('--seconds', dest='seconds', type=int, - default=DEFAULT_SECONDS, help=_("No of seconds")) - parser.add_argument('--username', dest='username', default=None, - help=_("Username")) + parser.add_argument( + "--days", dest="days", type=int, default=0, help=_("No of days") + ) + parser.add_argument( + "--seconds", + dest="seconds", + type=int, + default=DEFAULT_SECONDS, + help=_("No of seconds"), + ) + parser.add_argument( + "--username", dest="username", default=None, help=_("Username") + ) + # pylint: disable=unused-argument def handle(self, *args, **options): - days = int(options['days']) if 'days' in options else 0 - seconds = int(options['seconds']) if 'seconds' in options else \ - DEFAULT_SECONDS - username = options['username'] if 'username' in options else None + days = int(options["days"]) if "days" in options else 0 + seconds = int(options["seconds"]) if "seconds" in options else DEFAULT_SECONDS + username = options["username"] if "username" in options else None if username: self._reapply_perms(username=str(username)) diff --git a/onadata/apps/logger/management/commands/recover_deleted_attachments.py b/onadata/apps/logger/management/commands/recover_deleted_attachments.py index fa1bc6629f..3199f68ec1 100644 --- a/onadata/apps/logger/management/commands/recover_deleted_attachments.py +++ b/onadata/apps/logger/management/commands/recover_deleted_attachments.py @@ -18,23 +18,22 @@ def recover_deleted_attachments(form_id: str, stdout=None): :param: (str) form_id: Unique identifier for an XForm object :param: (sys.stdout) stdout: Python standard output. Default: None """ - instances = Instance.objects.filter( - xform__id=form_id, deleted_at__isnull=True) + instances = Instance.objects.filter(xform__id=form_id, deleted_at__isnull=True) for instance in instances: expected_attachments = instance.get_expected_media() - if not instance.attachments.filter( - deleted_at__isnull=True).count() == len(expected_attachments): + if not instance.attachments.filter(deleted_at__isnull=True).count() == len( + expected_attachments + ): attachments_to_recover = instance.attachments.filter( - deleted_at__isnull=False, - name__in=expected_attachments) + deleted_at__isnull=False, name__in=expected_attachments + ) for attachment in attachments_to_recover: attachment.deleted_at = None attachment.deleted_by = None attachment.save() if stdout: - stdout.write( - f'Recovered {attachment.name} ID: {attachment.id}') + stdout.write(f"Recovered {attachment.name} ID: {attachment.id}") # Regenerate instance JSON instance.json = instance.get_full_dict(load_existing=False) instance.save() @@ -45,11 +44,12 @@ class Command(BaseCommand): Management command used to recover wrongfully deleted attachments. """ - help = 'Restore wrongly deleted attachments' + + help = "Restore wrongly deleted attachments" def add_arguments(self, parser): - parser.add_argument('-f', '--form', dest='form_id', type=int) + parser.add_argument("-f", "--form", dest="form_id", type=int) def handle(self, *args, **options): - form_id = options.get('form_id') + form_id = options.get("form_id") recover_deleted_attachments(form_id, self.stdout) diff --git a/onadata/apps/logger/management/commands/remove_columns_from_briefcase_data.py b/onadata/apps/logger/management/commands/remove_columns_from_briefcase_data.py index ac99aba59a..a9a8fc3702 100644 --- a/onadata/apps/logger/management/commands/remove_columns_from_briefcase_data.py +++ b/onadata/apps/logger/management/commands/remove_columns_from_briefcase_data.py @@ -7,13 +7,12 @@ from onadata.apps.logger.xform_instance_parser import clean_and_parse_xml -def _traverse_child_nodes_and_delete_column( - xml_obj, column: str) -> None: +def _traverse_child_nodes_and_delete_column(xml_obj, column: str) -> None: childNodes = xml_obj.childNodes for elem in childNodes: if elem.nodeName in column: xml_obj.removeChild(elem) - if hasattr(elem, 'childNodes'): + if hasattr(elem, "childNodes"): _traverse_child_nodes_and_delete_column(elem, column) @@ -25,48 +24,45 @@ def remove_columns_from_xml(xml: str, columns: List[str]) -> str: class Command(BaseCommand): - help = _( - 'Delete specific columns from submission ' - 'XMLs pulled by ODK Briefcase.') + help = _("Delete specific columns from submission " "XMLs pulled by ODK Briefcase.") def add_arguments(self, parser): parser.add_argument( - '--input', - '-i', - dest='in_dir', - help='Path to instances directory to pull submission XMLs from.' + "--input", + "-i", + dest="in_dir", + help="Path to instances directory to pull submission XMLs from.", ) parser.add_argument( - '--output', - '-o', - default='replaced-submissions', - dest='out_dir', - help='Path to directory to output modified submission XMLs' + "--output", + "-o", + default="replaced-submissions", + dest="out_dir", + help="Path to directory to output modified submission XMLs", ) parser.add_argument( - '--columns', - '-c', - dest='columns', - help='Comma separated list of columns to remove from the XMLs' + "--columns", + "-c", + dest="columns", + help="Comma separated list of columns to remove from the XMLs", ) parser.add_argument( - '--overwrite', - '-f', + "--overwrite", + "-f", default=False, - dest='overwrite', - action='store_true', - help='Whether to overwrite the original submission' + dest="overwrite", + action="store_true", + help="Whether to overwrite the original submission", ) def handle(self, *args, **options): - columns: List[str] = options.get('columns').split(',') - in_dir: str = options.get('in_dir') - out_dir: str = options.get('out_dir') - overwrite: bool = options.get('overwrite') + columns: List[str] = options.get("columns").split(",") + in_dir: str = options.get("in_dir") + out_dir: str = options.get("out_dir") + overwrite: bool = options.get("overwrite") submission_folders = [ - xml_file for xml_file in os.listdir(in_dir) - if xml_file.startswith('uuid') + xml_file for xml_file in os.listdir(in_dir) if xml_file.startswith("uuid") ] total_files = len(submission_folders) modified_files = 0 @@ -76,36 +72,33 @@ def handle(self, *args, **options): for count, submission_folder in enumerate(submission_folders, start=1): self.stdout.write( - f'Modifying {submission_folder}. ' - f'Progress {count}/{total_files}') + f"Modifying {submission_folder}. " f"Progress {count}/{total_files}" + ) data = None - with open( - f'{in_dir}/{submission_folder}/submission.xml', - 'r') as in_file: - data = in_file.read().replace('\n', '') + with open(f"{in_dir}/{submission_folder}/submission.xml", "r") as in_file: + data = in_file.read().replace("\n", "") data = remove_columns_from_xml(data, columns) in_file.close() remove_columns_from_xml(data, columns) if not overwrite: - os.makedirs(f'{out_dir}/{submission_folder}') + os.makedirs(f"{out_dir}/{submission_folder}") with open( - f'{out_dir}/{submission_folder}/submission.xml', - 'w') as out_file: + f"{out_dir}/{submission_folder}/submission.xml", "w" + ) as out_file: out_file.write(data) out_file.close() else: with open( - f'{in_dir}/{submission_folder}/submission.xml', - 'r+')as out_file: + f"{in_dir}/{submission_folder}/submission.xml", "r+" + ) as out_file: out_file.truncate(0) out_file.write(data) out_file.close() modified_files += 1 - self.stdout.write( - f'Operation completed. Modified {modified_files} files.') + self.stdout.write(f"Operation completed. Modified {modified_files} files.") diff --git a/onadata/apps/logger/management/commands/replace_form_id_root_node.py b/onadata/apps/logger/management/commands/replace_form_id_root_node.py index 9d928ac88a..d4f5f47562 100644 --- a/onadata/apps/logger/management/commands/replace_form_id_root_node.py +++ b/onadata/apps/logger/management/commands/replace_form_id_root_node.py @@ -16,7 +16,8 @@ def replace_form_id_with_correct_root_node( - inst_id: int, root: str = None, commit: bool = False) -> str: + inst_id: int, root: str = None, commit: bool = False +) -> str: inst: Instance = Instance.objects.get(id=inst_id, deleted_at__isnull=True) initial_xml = inst.xml form_id = re.escape(inst.xform.id_string) @@ -25,8 +26,8 @@ def replace_form_id_with_correct_root_node( opening_tag_regex = f"<{form_id}" closing_tag_regex = f"" - edited_xml = re.sub(opening_tag_regex, f'<{root}', initial_xml) - edited_xml = re.sub(closing_tag_regex, f'', edited_xml) + edited_xml = re.sub(opening_tag_regex, f"<{root}", initial_xml) + edited_xml = re.sub(closing_tag_regex, f"", edited_xml) if commit: last_edited = timezone.now() @@ -36,7 +37,7 @@ def replace_form_id_with_correct_root_node( xform_instance=inst, ) inst.last_edited = last_edited - inst.checksum = sha256(edited_xml.encode('utf-8')).hexdigest() + inst.checksum = sha256(edited_xml.encode("utf-8")).hexdigest() inst.xml = edited_xml inst.save() return f"Modified Instance ID {inst.id} - History object {history.id}" @@ -49,39 +50,40 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( - '--instance-ids', - '-i', - dest='instance_ids', - help='Comma-separated list of instance ids.' + "--instance-ids", + "-i", + dest="instance_ids", + help="Comma-separated list of instance ids.", ) parser.add_argument( - '--commit-changes', - '-c', - action='store_true', - dest='commit', + "--commit-changes", + "-c", + action="store_true", + dest="commit", default=False, - help='Save XML changes' + help="Save XML changes", ) parser.add_argument( - '--root-node', - '-r', - dest='root', + "--root-node", + "-r", + dest="root", default=None, - help='Default root node name to replace the form ID with' + help="Default root node name to replace the form ID with", ) def handle(self, *args, **options): - instance_ids = options.get('instance_ids').split(',') - commit = options.get('commit') - root = options.get('root') + instance_ids = options.get("instance_ids").split(",") + commit = options.get("commit") + root = options.get("root") if not instance_ids: - raise CommandError('No instance id provided.') + raise CommandError("No instance id provided.") for inst_id in instance_ids: try: msg = replace_form_id_with_correct_root_node( - inst_id, root=root, commit=commit) + inst_id, root=root, commit=commit + ) except Instance.DoesNotExist: msg = f"Instance with ID {inst_id} does not exist" diff --git a/onadata/apps/logger/management/commands/restore_backup.py b/onadata/apps/logger/management/commands/restore_backup.py index 72d2552f18..b43fdb82e3 100644 --- a/onadata/apps/logger/management/commands/restore_backup.py +++ b/onadata/apps/logger/management/commands/restore_backup.py @@ -1,38 +1,46 @@ +# -*- coding: utf-8 -*- +""" +restore_backup command - Restore a zip backup of a form and all its submissions +""" import os import sys -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import ugettext_lazy, ugettext as _ +from django.utils.translation import gettext_lazy, gettext as _ from onadata.libs.utils.backup_tools import restore_backup_from_zip class Command(BaseCommand): - args = 'username input_file' - help = ugettext_lazy("Restore a zip backup of a form and all its" - " submissions") + """ + restore_backup command - Restore a zip backup of a form and all its submissions + """ + + args = "username input_file" + help = gettext_lazy("Restore a zip backup of a form and all its submissions") def handle(self, *args, **options): + # pylint: disable=invalid-name + User = get_user_model() # noqa N806 try: username = args[0] - except IndexError: - raise CommandError(_("You must provide the username to publish the" - " form to.")) + except IndexError as e: + raise CommandError( + _("You must provide the username to publish the form to.") + ) from e # make sure user exists try: User.objects.get(username=username) - except User.DoesNotExist: - raise CommandError(_("The user '%s' does not exist.") % username) + except User.DoesNotExist as e: + raise CommandError(_(f"The user '{username}' does not exist.")) from e try: input_file = args[1] - except IndexError: - raise CommandError(_("You must provide the path to restore from.")) + except IndexError as e: + raise CommandError(_("You must provide the path to restore from.")) from e else: input_file = os.path.realpath(input_file) - num_instances, num_restored = restore_backup_from_zip( - input_file, username) - sys.stdout.write("Restored %d of %d submissions\n" % - (num_restored, num_instances)) + num_instances, num_restored = restore_backup_from_zip(input_file, username) + sys.stdout.write(f"Restored {num_restored} of {num_instances } submissions\n") diff --git a/onadata/apps/logger/management/commands/set_xform_surveys_with_geopoints.py b/onadata/apps/logger/management/commands/set_xform_surveys_with_geopoints.py index dbb106045d..b40780779a 100644 --- a/onadata/apps/logger/management/commands/set_xform_surveys_with_geopoints.py +++ b/onadata/apps/logger/management/commands/set_xform_surveys_with_geopoints.py @@ -2,14 +2,14 @@ # vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models.xform import XForm from onadata.libs.utils.model_tools import queryset_iterator class Command(BaseCommand): - help = ugettext_lazy("Import a folder of XForms for ODK.") + help = gettext_lazy("Import a folder of XForms for ODK.") def handle(self, *args, **kwargs): xforms = XForm.objects.all() diff --git a/onadata/apps/logger/management/commands/set_xform_surveys_with_osm.py b/onadata/apps/logger/management/commands/set_xform_surveys_with_osm.py index 79b5664c87..df351dddae 100644 --- a/onadata/apps/logger/management/commands/set_xform_surveys_with_osm.py +++ b/onadata/apps/logger/management/commands/set_xform_surveys_with_osm.py @@ -2,7 +2,7 @@ # vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models.attachment import Attachment from onadata.apps.logger.models.xform import XForm @@ -10,13 +10,16 @@ class Command(BaseCommand): - help = ugettext_lazy("Set xform.instances_with_osm") + help = gettext_lazy("Set xform.instances_with_osm") def handle(self, *args, **kwargs): - pks = Attachment.objects.filter( - extension=Attachment.OSM, - instance__xform__instances_with_osm=False)\ - .values_list('instance__xform', flat=True).distinct() + pks = ( + Attachment.objects.filter( + extension=Attachment.OSM, instance__xform__instances_with_osm=False + ) + .values_list("instance__xform", flat=True) + .distinct() + ) xforms = XForm.objects.filter(pk__in=pks) total = xforms.count() count = 0 diff --git a/onadata/apps/logger/management/commands/sync_deleted_instances_fix.py b/onadata/apps/logger/management/commands/sync_deleted_instances_fix.py index a53dff842a..f80f9682be 100644 --- a/onadata/apps/logger/management/commands/sync_deleted_instances_fix.py +++ b/onadata/apps/logger/management/commands/sync_deleted_instances_fix.py @@ -6,36 +6,38 @@ from django.core.management import BaseCommand from django.utils import timezone from django.utils.dateparse import parse_datetime -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models import Instance class Command(BaseCommand): - help = ugettext_lazy("Fixes deleted instances by syncing " - "deleted items from mongo.") + help = gettext_lazy( + "Fixes deleted instances by syncing " "deleted items from mongo." + ) def handle(self, *args, **kwargs): # Reset all sql deletes to None - Instance.objects.exclude( - deleted_at=None, xform__downloadable=True).update(deleted_at=None) + Instance.objects.exclude(deleted_at=None, xform__downloadable=True).update( + deleted_at=None + ) # Get all mongo deletes - query = '{"$and": [{"_deleted_at": {"$exists": true}}, ' \ - '{"_deleted_at": {"$ne": null}}]}' + query = ( + '{"$and": [{"_deleted_at": {"$exists": true}}, ' + '{"_deleted_at": {"$ne": null}}]}' + ) query = json.loads(query) xform_instances = settings.MONGO_DB.instances cursor = xform_instances.find(query) for record in cursor: # update sql instance with deleted_at datetime from mongo try: - i = Instance.objects.get( - uuid=record["_uuid"], xform__downloadable=True) + i = Instance.objects.get(uuid=record["_uuid"], xform__downloadable=True) except Instance.DoesNotExist: continue else: deleted_at = parse_datetime(record["_deleted_at"]) if not timezone.is_aware(deleted_at): - deleted_at = timezone.make_aware( - deleted_at, timezone.utc) + deleted_at = timezone.make_aware(deleted_at, timezone.utc) i.set_deleted(deleted_at) diff --git a/onadata/apps/logger/management/commands/transferproject.py b/onadata/apps/logger/management/commands/transferproject.py index 56d86e9a2d..1f8548237a 100644 --- a/onadata/apps/logger/management/commands/transferproject.py +++ b/onadata/apps/logger/management/commands/transferproject.py @@ -4,8 +4,9 @@ from django.db import transaction from onadata.apps.logger.models import Project, XForm, DataView, MergedXForm -from onadata.apps.logger.models.project import set_object_permissions \ - as set_project_permissions +from onadata.apps.logger.models.project import ( + set_object_permissions as set_project_permissions, +) from onadata.libs.utils.project_utils import set_project_perms_to_xform @@ -19,35 +20,36 @@ class Command(BaseCommand): Depending on what is supplied for --project-id or --all-projects, the command will either transfer a single project or all the projects. """ - help = 'A command to reassign a project(s) from one user to the other.' + + help = "A command to reassign a project(s) from one user to the other." errors = [] def add_arguments(self, parser): parser.add_argument( - '--current-owner', - dest='current_owner', + "--current-owner", + dest="current_owner", type=str, - help='Username of the current owner of the project(s)', + help="Username of the current owner of the project(s)", ) parser.add_argument( - '--new-owner', - dest='new_owner', + "--new-owner", + dest="new_owner", type=str, - help='Username of the new owner of the project(s)', + help="Username of the new owner of the project(s)", ) parser.add_argument( - '--project-id', - dest='project_id', + "--project-id", + dest="project_id", type=int, - help='Id of the project to be transferred.', + help="Id of the project to be transferred.", ) parser.add_argument( - '--all-projects', - dest='all_projects', - action='store_true', - help='Supply this command if all the projects are to be' - ' transferred. If not, do not include the argument', + "--all-projects", + dest="all_projects", + action="store_true", + help="Supply this command if all the projects are to be" + " transferred. If not, do not include the argument", ) def get_user(self, username): # pylint: disable=C0111 @@ -65,7 +67,8 @@ def update_xform_with_new_user(self, project, user): for the xForm and the project. """ xforms = XForm.objects.filter( - project=project, deleted_at__isnull=True, downloadable=True) + project=project, deleted_at__isnull=True, downloadable=True + ) for form in xforms: form.user = user form.created_by = user @@ -75,9 +78,10 @@ def update_xform_with_new_user(self, project, user): @staticmethod def update_data_views(form): - """Update DataView project for the XForm given. """ + """Update DataView project for the XForm given.""" dataviews = DataView.objects.filter( - xform=form, project=form.project, deleted_at__isnull=True) + xform=form, project=form.project, deleted_at__isnull=True + ) for data_view in dataviews: data_view.project = form.project data_view.save() @@ -86,7 +90,8 @@ def update_data_views(form): def update_merged_xform(project, user): """Update ownership of MergedXforms.""" merged_xforms = MergedXForm.objects.filter( - project=project, deleted_at__isnull=True) + project=project, deleted_at__isnull=True + ) for form in merged_xforms: form.user = user form.created_by = user @@ -96,13 +101,13 @@ def update_merged_xform(project, user): @transaction.atomic() def handle(self, *args, **options): """Transfer projects from one user to another.""" - from_user = self.get_user(options['current_owner']) - to_user = self.get_user(options['new_owner']) - project_id = options.get('project_id') - transfer_all_projects = options.get('all_projects') + from_user = self.get_user(options["current_owner"]) + to_user = self.get_user(options["new_owner"]) + project_id = options.get("project_id") + transfer_all_projects = options.get("all_projects") if self.errors: - self.stdout.write(''.join(self.errors)) + self.stdout.write("".join(self.errors)) return # No need to validate project ownership as they filtered @@ -110,10 +115,10 @@ def handle(self, *args, **options): projects = [] if transfer_all_projects: projects = Project.objects.filter( - organization=from_user, deleted_at__isnull=True) + organization=from_user, deleted_at__isnull=True + ) else: - projects = Project.objects.filter( - id=project_id, organization=from_user) + projects = Project.objects.filter(id=project_id, organization=from_user) for project in projects: project.organization = to_user @@ -124,4 +129,4 @@ def handle(self, *args, **options): self.update_merged_xform(project, to_user) set_project_permissions(Project, project, created=True) - self.stdout.write('Projects transferred successfully') + self.stdout.write("Projects transferred successfully") diff --git a/onadata/apps/logger/management/commands/update_moved_forms.py b/onadata/apps/logger/management/commands/update_moved_forms.py index 9e10774134..666c76c251 100644 --- a/onadata/apps/logger/management/commands/update_moved_forms.py +++ b/onadata/apps/logger/management/commands/update_moved_forms.py @@ -1,30 +1,31 @@ from django.core.management import BaseCommand -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models.project import Project from onadata.libs.utils.model_tools import queryset_iterator class Command(BaseCommand): - help = ugettext_lazy("Ensures all the forms are owned by the project" - " owner") + help = gettext_lazy("Ensures all the forms are owned by the project" " owner") def handle(self, *args, **kwargs): - self.stdout.write("Updating forms owner", ending='\n') + self.stdout.write("Updating forms owner", ending="\n") for project in queryset_iterator(Project.objects.all()): for xform in project.xform_set.all(): try: if xform.user != project.organization: self.stdout.write( - "Processing: {} - {}".format(xform.id_string, - xform.user.username) + "Processing: {} - {}".format( + xform.id_string, xform.user.username + ) ) xform.user = project.organization xform.save() except Exception: self.stdout.write( - "Error processing: {} - {}".format(xform.id_string, - xform.user.username) + "Error processing: {} - {}".format( + xform.id_string, xform.user.username + ) ) pass diff --git a/onadata/apps/logger/management/commands/update_xform_uuids.py b/onadata/apps/logger/management/commands/update_xform_uuids.py index d6b7c2db9a..ecc0ed9c8c 100644 --- a/onadata/apps/logger/management/commands/update_xform_uuids.py +++ b/onadata/apps/logger/management/commands/update_xform_uuids.py @@ -1,31 +1,40 @@ #!/usr/bin/env python # vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 +# -*- coding=utf-8 -*- +""" +update_xform_uuids command - Set uuid from a CSV file +""" import csv from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy -from onadata.apps.logger.models.xform import (DuplicateUUIDError, XForm, - update_xform_uuid) +from onadata.apps.logger.models.xform import ( + DuplicateUUIDError, + XForm, + update_xform_uuid, +) class Command(BaseCommand): - help = ugettext_lazy( - "Use a csv file with username, id_string and new_uuid to set new" - " uuids") + """Use a csv file with username, id_string and new_uuid to set new uuids.""" + + help = gettext_lazy( + "Use a csv file with username, id_string and new_uuid to set new uuids" + ) def add_arguments(self, parser): - parser.add_argument( - '-f', '--file', help=ugettext_lazy("Path to csv file")) + parser.add_argument("-f", "--file", help=gettext_lazy("Path to csv file")) def handle(self, *args, **kwargs): + """Use a csv file with username, id_string and new_uuid to set new uuids.""" # all options are required - if not kwargs.get('file'): + if not kwargs.get("file"): raise CommandError("You must provide a path to the csv file") # try open the file try: - with open(kwargs.get('file'), "r") as f: + with open(kwargs.get("file"), "r", encoding="utf-8") as f: lines = csv.reader(f) i = 0 for line in lines: @@ -35,18 +44,16 @@ def handle(self, *args, **kwargs): uuid = line[2] update_xform_uuid(username, id_string, uuid) except IndexError: - self.stderr.write( - "line %d is in an invalid format" % (i + 1)) + self.stderr.write(f"line {i + 1} is in an invalid format") except XForm.DoesNotExist: - self.stderr.write("XForm with username: %s and id " - "string: %s does not exist" - % (username, id_string)) - except DuplicateUUIDError: self.stderr.write( - "An xform with uuid: %s already exists" % uuid) + f"XForm with username: {username} and id " + f"string: {id_string} does not exist" + ) + except DuplicateUUIDError: + self.stderr.write(f"An xform with uuid: {uuid} already exists") else: i += 1 - self.stdout.write("Updated %d rows" % i) - except IOError: - raise CommandError( - "file %s could not be open" % kwargs.get('file')) + self.stdout.write(f"Updated {i} rows") + except IOError as e: + raise CommandError(f"file {kwargs.get('file')} could not be open") from e diff --git a/onadata/apps/logger/migrations/0001_initial.py b/onadata/apps/logger/migrations/0001_initial.py index e1d3f4fbc8..f102ed93b5 100644 --- a/onadata/apps/logger/migrations/0001_initial.py +++ b/onadata/apps/logger/migrations/0001_initial.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals from django.db import models, migrations -import jsonfield.fields import onadata.apps.logger.models.xform import django.contrib.gis.db.models.fields from django.conf import settings @@ -13,327 +12,452 @@ class Migration(migrations.Migration): dependencies = [ - ('taggit', '0001_initial'), + ("taggit", "0001_initial"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('contenttypes', '0001_initial'), + ("contenttypes", "0001_initial"), ] operations = [ migrations.CreateModel( - name='Attachment', + name="Attachment", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('media_file', models.FileField( - max_length=255, - upload_to=onadata.apps.logger.models.attachment.upload_to) - ), - ('mimetype', models.CharField( - default=b'', max_length=50, blank=True)), - ('extension', - models.CharField(default='non', max_length=10, db_index=True) - ), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ( + "media_file", + models.FileField( + max_length=255, + upload_to=onadata.apps.logger.models.attachment.upload_to, + ), + ), + ("mimetype", models.CharField(default=b"", max_length=50, blank=True)), + ( + "extension", + models.CharField(default="non", max_length=10, db_index=True), + ), ], - options={ - }, + options={}, bases=(models.Model,), ), migrations.CreateModel( - name='DataView', + name="DataView", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('name', models.CharField(max_length=255)), - ('columns', jsonfield.fields.JSONField()), - ('query', jsonfield.fields.JSONField(default={}, blank=True)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("name", models.CharField(max_length=255)), + ("columns", models.JSONField()), + ("query", models.JSONField(default={}, blank=True)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), ], options={ - 'verbose_name': 'Data View', - 'verbose_name_plural': 'Data Views', + "verbose_name": "Data View", + "verbose_name_plural": "Data Views", }, bases=(models.Model,), ), migrations.CreateModel( - name='Instance', + name="Instance", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('json', jsonfield.fields.JSONField(default={})), - ('xml', models.TextField()), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('deleted_at', models.DateTimeField(default=None, null=True)), - ('status', models.CharField(default='submitted_via_web', - max_length=20)), - ('uuid', models.CharField(default='', max_length=249)), - ('version', models.CharField(max_length=255, null=True)), - ('geom', - django.contrib.gis.db.models.fields.GeometryCollectionField( - srid=4326, null=True)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("json", models.JSONField(default={})), + ("xml", models.TextField()), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ("deleted_at", models.DateTimeField(default=None, null=True)), + ( + "status", + models.CharField(default="submitted_via_web", max_length=20), + ), + ("uuid", models.CharField(default="", max_length=249)), + ("version", models.CharField(max_length=255, null=True)), + ( + "geom", + django.contrib.gis.db.models.fields.GeometryCollectionField( + srid=4326, null=True + ), + ), ], - options={ - }, + options={}, bases=(models.Model,), ), migrations.CreateModel( - name='InstanceHistory', + name="InstanceHistory", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('xml', models.TextField()), - ('uuid', models.CharField(default='', max_length=249)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('xform_instance', - models.ForeignKey(related_name='submission_history', - to='logger.Instance', - on_delete=models.CASCADE)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("xml", models.TextField()), + ("uuid", models.CharField(default="", max_length=249)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "xform_instance", + models.ForeignKey( + related_name="submission_history", + to="logger.Instance", + on_delete=models.CASCADE, + ), + ), ], - options={ - }, + options={}, bases=(models.Model,), ), migrations.CreateModel( - name='Note', + name="Note", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('note', models.TextField()), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('instance', - models.ForeignKey( - related_name='notes', to='logger.Instance', - on_delete=models.CASCADE) - ), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("note", models.TextField()), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "instance", + models.ForeignKey( + related_name="notes", + to="logger.Instance", + on_delete=models.CASCADE, + ), + ), ], options={ - 'permissions': (('view_note', 'View note'),), + "permissions": (("view_note", "View note"),), }, bases=(models.Model,), ), migrations.CreateModel( - name='Project', + name="Project", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('name', models.CharField(max_length=255)), - ('metadata', jsonfield.fields.JSONField(blank=True)), - ('shared', models.BooleanField(default=False)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('created_by', - models.ForeignKey(related_name='project_owner', - to=settings.AUTH_USER_MODEL, - on_delete=models.CASCADE)), - ('organization', models.ForeignKey( - related_name='project_org', to=settings.AUTH_USER_MODEL, - on_delete=models.CASCADE)), - ('tags', - taggit.managers.TaggableManager( - to='taggit.Tag', through='taggit.TaggedItem', - help_text='A comma-separated list of tags.', - verbose_name='Tags')), - ('user_stars', - models.ManyToManyField(related_name='project_stars', - to=settings.AUTH_USER_MODEL)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("name", models.CharField(max_length=255)), + ("metadata", models.JSONField(blank=True)), + ("shared", models.BooleanField(default=False)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "created_by", + models.ForeignKey( + related_name="project_owner", + to=settings.AUTH_USER_MODEL, + on_delete=models.CASCADE, + ), + ), + ( + "organization", + models.ForeignKey( + related_name="project_org", + to=settings.AUTH_USER_MODEL, + on_delete=models.CASCADE, + ), + ), + ( + "tags", + taggit.managers.TaggableManager( + to="taggit.Tag", + through="taggit.TaggedItem", + help_text="A comma-separated list of tags.", + verbose_name="Tags", + ), + ), + ( + "user_stars", + models.ManyToManyField( + related_name="project_stars", to=settings.AUTH_USER_MODEL + ), + ), ], options={ - 'permissions': ( - ('view_project', 'Can view project'), - ('add_project_xform', 'Can add xform to project'), - ('report_project_xform', - 'Can make submissions to the project'), - ('transfer_project', - 'Can transfer project to different owner'), - ('can_export_project_data', 'Can export data in project')), + "permissions": ( + ("view_project", "Can view project"), + ("add_project_xform", "Can add xform to project"), + ("report_project_xform", "Can make submissions to the project"), + ("transfer_project", "Can transfer project to different owner"), + ("can_export_project_data", "Can export data in project"), + ), }, bases=(models.Model,), ), migrations.CreateModel( - name='SurveyType', + name="SurveyType", fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('slug', models.CharField(unique=True, max_length=100)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("slug", models.CharField(unique=True, max_length=100)), ], - options={ - }, + options={}, bases=(models.Model,), ), migrations.CreateModel( - name='Widget', + name="Widget", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('object_id', models.PositiveIntegerField()), - ('widget_type', - models.CharField(default=b'charts', max_length=25, - choices=[(b'charts', b'Charts')])), - ('view_type', models.CharField(max_length=50)), - ('column', models.CharField(max_length=50)), - ('group_by', - models.CharField(default=None, max_length=50, null=True, - blank=True)), - ('title', - models.CharField(default=None, max_length=50, null=True, - blank=True)), - ('description', - models.CharField(default=None, max_length=255, - null=True, blank=True)), - ('key', - models.CharField(unique=True, max_length=32, db_index=True)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('content_type', - models.ForeignKey(to='contenttypes.ContentType', on_delete=models.CASCADE)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("object_id", models.PositiveIntegerField()), + ( + "widget_type", + models.CharField( + default=b"charts", + max_length=25, + choices=[(b"charts", b"Charts")], + ), + ), + ("view_type", models.CharField(max_length=50)), + ("column", models.CharField(max_length=50)), + ( + "group_by", + models.CharField( + default=None, max_length=50, null=True, blank=True + ), + ), + ( + "title", + models.CharField( + default=None, max_length=50, null=True, blank=True + ), + ), + ( + "description", + models.CharField( + default=None, max_length=255, null=True, blank=True + ), + ), + ("key", models.CharField(unique=True, max_length=32, db_index=True)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "content_type", + models.ForeignKey( + to="contenttypes.ContentType", on_delete=models.CASCADE + ), + ), ], - options={ - }, + options={}, bases=(models.Model,), ), migrations.CreateModel( - name='XForm', + name="XForm", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('xls', - models.FileField( - null=True, - upload_to=onadata.apps.logger.models.xform.upload_to)), - ('json', models.TextField(default='')), - ('description', models.TextField(default='', null=True, - blank=True)), - ('xml', models.TextField()), - ('require_auth', models.BooleanField(default=False)), - ('shared', models.BooleanField(default=False)), - ('shared_data', models.BooleanField(default=False)), - ('downloadable', models.BooleanField(default=True)), - ('allows_sms', models.BooleanField(default=False)), - ('encrypted', models.BooleanField(default=False)), - ('sms_id_string', - models.SlugField(default=b'', verbose_name='SMS ID', - max_length=100, editable=False)), - ('id_string', - models.SlugField(verbose_name='ID', max_length=100, - editable=False)), - ('title', models.CharField(max_length=255, editable=False)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('deleted_at', models.DateTimeField(null=True, blank=True)), - ('last_submission_time', - models.DateTimeField(null=True, blank=True)), - ('has_start_time', models.BooleanField(default=False)), - ('uuid', models.CharField(default='', max_length=32)), - ('bamboo_dataset', - models.CharField(default='', max_length=60)), - ('instances_with_geopoints', - models.BooleanField(default=False)), - ('instances_with_osm', models.BooleanField(default=False)), - ('num_of_submissions', models.IntegerField(default=0)), - ('version', - models.CharField(max_length=255, null=True, blank=True)), - ('created_by', - models.ForeignKey(blank=True, to=settings.AUTH_USER_MODEL, - null=True, on_delete=models.CASCADE)), - ('project', models.ForeignKey(to='logger.Project', - on_delete=models.CASCADE)), - ('tags', - taggit.managers.TaggableManager( - to='taggit.Tag', through='taggit.TaggedItem', - help_text='A comma-separated list of tags.', - verbose_name='Tags')), - ('user', - models.ForeignKey(related_name='xforms', - to=settings.AUTH_USER_MODEL, null=True, - on_delete=models.CASCADE)) + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ( + "xls", + models.FileField( + null=True, upload_to=onadata.apps.logger.models.xform.upload_to + ), + ), + ("json", models.TextField(default="")), + ("description", models.TextField(default="", null=True, blank=True)), + ("xml", models.TextField()), + ("require_auth", models.BooleanField(default=False)), + ("shared", models.BooleanField(default=False)), + ("shared_data", models.BooleanField(default=False)), + ("downloadable", models.BooleanField(default=True)), + ("allows_sms", models.BooleanField(default=False)), + ("encrypted", models.BooleanField(default=False)), + ( + "sms_id_string", + models.SlugField( + default=b"", + verbose_name="SMS ID", + max_length=100, + editable=False, + ), + ), + ( + "id_string", + models.SlugField(verbose_name="ID", max_length=100, editable=False), + ), + ("title", models.CharField(max_length=255, editable=False)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ("deleted_at", models.DateTimeField(null=True, blank=True)), + ("last_submission_time", models.DateTimeField(null=True, blank=True)), + ("has_start_time", models.BooleanField(default=False)), + ("uuid", models.CharField(default="", max_length=32)), + ("bamboo_dataset", models.CharField(default="", max_length=60)), + ("instances_with_geopoints", models.BooleanField(default=False)), + ("instances_with_osm", models.BooleanField(default=False)), + ("num_of_submissions", models.IntegerField(default=0)), + ("version", models.CharField(max_length=255, null=True, blank=True)), + ( + "created_by", + models.ForeignKey( + blank=True, + to=settings.AUTH_USER_MODEL, + null=True, + on_delete=models.CASCADE, + ), + ), + ( + "project", + models.ForeignKey(to="logger.Project", on_delete=models.CASCADE), + ), + ( + "tags", + taggit.managers.TaggableManager( + to="taggit.Tag", + through="taggit.TaggedItem", + help_text="A comma-separated list of tags.", + verbose_name="Tags", + ), + ), + ( + "user", + models.ForeignKey( + related_name="xforms", + to=settings.AUTH_USER_MODEL, + null=True, + on_delete=models.CASCADE, + ), + ), ], options={ - 'ordering': ('id_string',), - 'verbose_name': 'XForm', - 'verbose_name_plural': 'XForms', - 'permissions': ( - ('view_xform', 'Can view associated data'), - ('report_xform', 'Can make submissions to the form'), - ('move_xform', 'Can move form between projects'), - ('transfer_xform', 'Can transfer form ownership.'), - ('can_export_xform_data', 'Can export form data')), + "ordering": ("id_string",), + "verbose_name": "XForm", + "verbose_name_plural": "XForms", + "permissions": ( + ("view_xform", "Can view associated data"), + ("report_xform", "Can make submissions to the form"), + ("move_xform", "Can move form between projects"), + ("transfer_xform", "Can transfer form ownership."), + ("can_export_xform_data", "Can export form data"), + ), }, bases=(models.Model,), ), migrations.AlterUniqueTogether( - name='xform', + name="xform", unique_together=set( - [('user', 'id_string', 'project'), - ('user', 'sms_id_string', 'project')]), + [("user", "id_string", "project"), ("user", "sms_id_string", "project")] + ), ), migrations.AlterUniqueTogether( - name='project', - unique_together=set([('name', 'organization')]), + name="project", + unique_together=set([("name", "organization")]), ), migrations.AddField( - model_name='instance', - name='survey_type', - field=models.ForeignKey(to='logger.SurveyType', - on_delete=models.CASCADE), + model_name="instance", + name="survey_type", + field=models.ForeignKey(to="logger.SurveyType", on_delete=models.CASCADE), preserve_default=True, ), migrations.AddField( - model_name='instance', - name='tags', + model_name="instance", + name="tags", field=taggit.managers.TaggableManager( - to='taggit.Tag', through='taggit.TaggedItem', - help_text='A comma-separated list of tags.', - verbose_name='Tags'), + to="taggit.Tag", + through="taggit.TaggedItem", + help_text="A comma-separated list of tags.", + verbose_name="Tags", + ), preserve_default=True, ), migrations.AddField( - model_name='instance', - name='user', + model_name="instance", + name="user", field=models.ForeignKey( - related_name='instances', to=settings.AUTH_USER_MODEL, + related_name="instances", + to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE, - null=True), + null=True, + ), preserve_default=True, ), migrations.AddField( - model_name='instance', - name='xform', + model_name="instance", + name="xform", field=models.ForeignKey( - related_name='instances', - to='logger.XForm', null=True, on_delete=models.CASCADE), + related_name="instances", + to="logger.XForm", + null=True, + on_delete=models.CASCADE, + ), preserve_default=True, ), migrations.AddField( - model_name='dataview', - name='project', - field=models.ForeignKey(to='logger.Project', - on_delete=models.CASCADE), + model_name="dataview", + name="project", + field=models.ForeignKey(to="logger.Project", on_delete=models.CASCADE), preserve_default=True, ), migrations.AddField( - model_name='dataview', - name='xform', - field=models.ForeignKey(to='logger.XForm', - on_delete=models.CASCADE), + model_name="dataview", + name="xform", + field=models.ForeignKey(to="logger.XForm", on_delete=models.CASCADE), preserve_default=True, ), migrations.AddField( - model_name='attachment', - name='instance', + model_name="attachment", + name="instance", field=models.ForeignKey( - related_name='attachments',to='logger.Instance', - on_delete=models.CASCADE), + related_name="attachments", + to="logger.Instance", + on_delete=models.CASCADE, + ), preserve_default=True, ), ] diff --git a/onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py b/onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py new file mode 100644 index 0000000000..74bf18b414 --- /dev/null +++ b/onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py @@ -0,0 +1,1268 @@ +# Generated by Django 3.2.13 on 2022-04-25 06:54 + +import datetime +from django.conf import settings +import django.contrib.gis.db.models.fields +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion +from django.utils.timezone import utc +import onadata.apps.logger.models.attachment +import onadata.apps.logger.models.xform +import onadata.libs.utils.common_tools +from onadata.apps.logger.models import Instance +from onadata.libs.utils.model_tools import queryset_iterator +from onadata.libs.utils.logger_tools import create_xform_version +import taggit.managers +from hashlib import md5 + + +def recalculate_xform_hash(apps, schema_editor): # pylint: disable=W0613 + """ + Recalculate all XForm hashes. + """ + XForm = apps.get_model("logger", "XForm") # pylint: disable=C0103 + xforms = XForm.objects.filter(downloadable=True, deleted_at__isnull=True).only( + "xml" + ) + count = xforms.count() + counter = 0 + + for xform in queryset_iterator(xforms, 500): + hash_value = md5(xform.xml.encode("utf8")).hexdigest() + xform.hash = f"md5:{hash_value}" + xform.save(update_fields=["hash"]) + counter += 1 + if counter % 500 == 0: + print(f"Processed {counter} of {count} forms.") + + print(f"Processed {counter} forms.") + + +def generate_uuid_if_missing(apps, schema_editor): + """ + Generate uuids for XForms without them + """ + XForm = apps.get_model("logger", "XForm") + + for xform in XForm.objects.filter(uuid=""): + xform.uuid = onadata.libs.utils.common_tools.get_uuid() + xform.save() + + +def regenerate_instance_json(apps, schema_editor): + """ + Regenerate Instance JSON + """ + for inst in Instance.objects.filter( + deleted_at__isnull=True, + xform__downloadable=True, + xform__deleted_at__isnull=True, + ): + inst.json = inst.get_full_dict(load_existing=False) + inst.save() + + +def create_initial_xform_version(apps, schema_editor): + """ + Creates an XFormVersion object for an XForm that has no + Version + """ + queryset = onadata.apps.logger.models.xform.XForm.objects.filter( + downloadable=True, deleted_at__isnull=True + ) + for xform in queryset.iterator(): + if xform.version: + create_xform_version(xform, xform.user) + + +class Migration(migrations.Migration): + replaces = [ + ("logger", "0001_initial"), + ("logger", "0002_auto_20150717_0048"), + ("logger", "0003_dataview_instances_with_geopoints"), + ("logger", "0004_auto_20150910_0056"), + ("logger", "0005_auto_20151015_0758"), + ("logger", "0006_auto_20151106_0130"), + ("logger", "0007_osmdata_field_name"), + ("logger", "0008_osmdata_osm_type"), + ("logger", "0009_auto_20151111_0438"), + ("logger", "0010_attachment_file_size"), + ("logger", "0011_dataview_matches_parent"), + ("logger", "0012_auto_20160114_0708"), + ("logger", "0013_note_created_by"), + ("logger", "0014_note_instance_field"), + ("logger", "0015_auto_20160222_0559"), + ("logger", "0016_widget_aggregation"), + ("logger", "0017_auto_20160224_0130"), + ("logger", "0018_auto_20160301_0330"), + ("logger", "0019_auto_20160307_0256"), + ("logger", "0020_auto_20160408_0325"), + ("logger", "0021_auto_20160408_0919"), + ("logger", "0022_auto_20160418_0518"), + ("logger", "0023_auto_20160419_0403"), + ("logger", "0024_xform_has_hxl_support"), + ("logger", "0025_xform_last_updated_at"), + ("logger", "0026_auto_20160913_0239"), + ("logger", "0027_auto_20161201_0730"), + ("logger", "0028_auto_20170221_0838"), + ("logger", "0029_auto_20170221_0908"), + ("logger", "0030_auto_20170227_0137"), + ("logger", "0028_auto_20170217_0502"), + ("logger", "0031_merge"), + ("logger", "0032_project_deleted_at"), + ("logger", "0033_auto_20170705_0159"), + ("logger", "0034_mergedxform"), + ("logger", "0035_auto_20170712_0529"), + ("logger", "0036_xform_is_merged_dataset"), + ("logger", "0034_auto_20170814_0432"), + ("logger", "0037_merge_20170825_0238"), + ("logger", "0038_auto_20170828_1718"), + ("logger", "0039_auto_20170909_2052"), + ("logger", "0040_auto_20170912_1504"), + ("logger", "0041_auto_20170912_1512"), + ("logger", "0042_xform_hash"), + ("logger", "0043_auto_20171010_0403"), + ("logger", "0044_xform_hash_sql_update"), + ("logger", "0045_attachment_name"), + ("logger", "0046_auto_20180314_1618"), + ("logger", "0047_dataview_deleted_at"), + ("logger", "0048_dataview_deleted_by"), + ("logger", "0049_xform_deleted_by"), + ("logger", "0050_project_deleted_by"), + ("logger", "0051_auto_20180522_1118"), + ("logger", "0052_auto_20180805_2233"), + ("logger", "0053_submissionreview"), + ("logger", "0054_instance_has_a_review"), + ("logger", "0055_auto_20180904_0713"), + ("logger", "0056_auto_20190125_0517"), + ("logger", "0057_xform_public_key"), + ("logger", "0058_auto_20191211_0900"), + ("logger", "0059_attachment_deleted_by"), + ("logger", "0060_auto_20200305_0357"), + ("logger", "0061_auto_20200713_0814"), + ("logger", "0062_auto_20210202_0248"), + ("logger", "0063_xformversion"), + ("logger", "0064_auto_20210304_0314"), + ] + + initial = True + + dependencies = [ + ("contenttypes", "0002_remove_content_type_name"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("auth", "0001_initial"), + ("taggit", "0001_initial"), + ("contenttypes", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="Project", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=255)), + ( + "metadata", + django.contrib.postgres.fields.jsonb.JSONField(default=dict), + ), + ("shared", models.BooleanField(default=False)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "created_by", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="project_owner", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "organization", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="project_org", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "tags", + taggit.managers.TaggableManager( + help_text="A comma-separated list of tags.", + through="taggit.TaggedItem", + to="taggit.Tag", + verbose_name="Tags", + ), + ), + ( + "user_stars", + models.ManyToManyField( + related_name="project_stars", to=settings.AUTH_USER_MODEL + ), + ), + ("deleted_at", models.DateTimeField(blank=True, null=True)), + ], + options={ + "permissions": ( + ("view_project", "Can view project"), + ("add_project_xform", "Can add xform to project"), + ("report_project_xform", "Can make submissions to the project"), + ("transfer_project", "Can transfer project to different owner"), + ("can_export_project_data", "Can export data in project"), + ("view_project_all", "Can view all associated data"), + ("view_project_data", "Can view submitted data"), + ), + "unique_together": {("name", "organization")}, + }, + ), + migrations.CreateModel( + name="SurveyType", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("slug", models.CharField(max_length=100, unique=True)), + ], + ), + migrations.CreateModel( + name="XForm", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "xls", + models.FileField( + null=True, upload_to=onadata.apps.logger.models.xform.upload_to + ), + ), + ("json", models.TextField(default="")), + ("description", models.TextField(blank=True, default="", null=True)), + ("xml", models.TextField()), + ("require_auth", models.BooleanField(default=False)), + ("shared", models.BooleanField(default=False)), + ("shared_data", models.BooleanField(default=False)), + ("downloadable", models.BooleanField(default=True)), + ("allows_sms", models.BooleanField(default=False)), + ("encrypted", models.BooleanField(default=False)), + ( + "sms_id_string", + models.SlugField( + default=b"", + editable=False, + max_length=100, + verbose_name="SMS ID", + ), + ), + ( + "id_string", + models.SlugField(editable=False, max_length=100, verbose_name="ID"), + ), + ("title", models.CharField(editable=False, max_length=255)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ("deleted_at", models.DateTimeField(blank=True, null=True)), + ("last_submission_time", models.DateTimeField(blank=True, null=True)), + ("has_start_time", models.BooleanField(default=False)), + ("uuid", models.CharField(default="", max_length=32)), + ("bamboo_dataset", models.CharField(default="", max_length=60)), + ("instances_with_geopoints", models.BooleanField(default=False)), + ("instances_with_osm", models.BooleanField(default=False)), + ("num_of_submissions", models.IntegerField(default=0)), + ("version", models.CharField(blank=True, max_length=255, null=True)), + ( + "created_by", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "project", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="logger.project" + ), + ), + ( + "tags", + taggit.managers.TaggableManager( + help_text="A comma-separated list of tags.", + through="taggit.TaggedItem", + to="taggit.Tag", + verbose_name="Tags", + ), + ), + ( + "user", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="xforms", + to=settings.AUTH_USER_MODEL, + ), + ), + ("has_hxl_support", models.BooleanField(default=False)), + ( + "last_updated_at", + models.DateTimeField( + auto_now=True, + default=datetime.datetime( + 2016, 8, 18, 12, 43, 30, 316792, tzinfo=utc + ), + ), + ), + ("is_merged_dataset", models.BooleanField(default=False)), + ( + "hash", + models.CharField( + blank=True, + default=None, + max_length=36, + null=True, + verbose_name="Hash", + ), + ), + ], + options={ + "ordering": ("pk",), + "verbose_name": "XForm", + "verbose_name_plural": "XForms", + "permissions": ( + ("view_xform", "Can view associated data"), + ("view_xform_all", "Can view all associated data"), + ("view_xform_data", "Can view submitted data"), + ("report_xform", "Can make submissions to the form"), + ("move_xform", "Can move form between projects"), + ("transfer_xform", "Can transfer form ownership."), + ("can_export_xform_data", "Can export form data"), + ("delete_submission", "Can delete submissions from form"), + ), + "unique_together": { + ("user", "id_string", "project"), + ("user", "sms_id_string", "project"), + }, + }, + ), + migrations.CreateModel( + name="Instance", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("json", django.contrib.postgres.fields.jsonb.JSONField(default=dict)), + ("xml", models.TextField()), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ("deleted_at", models.DateTimeField(default=None, null=True)), + ( + "status", + models.CharField(default="submitted_via_web", max_length=20), + ), + ("uuid", models.CharField(db_index=True, default="", max_length=249)), + ("version", models.CharField(max_length=255, null=True)), + ( + "geom", + django.contrib.gis.db.models.fields.GeometryCollectionField( + null=True, srid=4326 + ), + ), + ( + "survey_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="logger.surveytype", + ), + ), + ( + "tags", + taggit.managers.TaggableManager( + help_text="A comma-separated list of tags.", + through="taggit.TaggedItem", + to="taggit.Tag", + verbose_name="Tags", + ), + ), + ( + "user", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="instances", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "xform", + models.ForeignKey( + default=328, + on_delete=django.db.models.deletion.CASCADE, + related_name="instances", + to="logger.xform", + ), + ), + ("last_edited", models.DateTimeField(default=None, null=True)), + ( + "media_all_received", + models.NullBooleanField( + default=True, verbose_name="Received All Media Attachemts" + ), + ), + ( + "media_count", + models.PositiveIntegerField( + default=0, null=True, verbose_name="Received Media Attachments" + ), + ), + ( + "total_media", + models.PositiveIntegerField( + default=0, null=True, verbose_name="Total Media Attachments" + ), + ), + ( + "checksum", + models.CharField( + blank=True, db_index=True, max_length=64, null=True + ), + ), + ], + options={ + "unique_together": {("xform", "uuid")}, + }, + ), + migrations.CreateModel( + name="ProjectUserObjectPermission", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "content_object", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="logger.project" + ), + ), + ( + "permission", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="auth.permission", + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "abstract": False, + "unique_together": {("user", "permission", "content_object")}, + }, + ), + migrations.CreateModel( + name="ProjectGroupObjectPermission", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "content_object", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="logger.project" + ), + ), + ( + "group", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="auth.group" + ), + ), + ( + "permission", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="auth.permission", + ), + ), + ], + options={ + "abstract": False, + "unique_together": {("group", "permission", "content_object")}, + }, + ), + migrations.CreateModel( + name="XFormUserObjectPermission", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "content_object", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="logger.xform" + ), + ), + ( + "permission", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="auth.permission", + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "abstract": False, + "unique_together": {("user", "permission", "content_object")}, + }, + ), + migrations.CreateModel( + name="XFormGroupObjectPermission", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "content_object", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="logger.xform" + ), + ), + ( + "group", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="auth.group" + ), + ), + ( + "permission", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="auth.permission", + ), + ), + ], + options={ + "abstract": False, + "unique_together": {("group", "permission", "content_object")}, + }, + ), + migrations.CreateModel( + name="Note", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("note", models.TextField()), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "instance", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="notes", + to="logger.instance", + ), + ), + ( + "created_by", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ("instance_field", models.TextField(blank=True, null=True)), + ], + options={ + "permissions": (("view_note", "View note"),), + }, + ), + migrations.CreateModel( + name="DataView", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=255)), + ("columns", django.contrib.postgres.fields.jsonb.JSONField()), + ( + "query", + django.contrib.postgres.fields.jsonb.JSONField( + blank=True, default=dict + ), + ), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "project", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="logger.project" + ), + ), + ( + "xform", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="logger.xform" + ), + ), + ("instances_with_geopoints", models.BooleanField(default=False)), + ("matches_parent", models.BooleanField(default=False)), + ], + options={ + "verbose_name": "Data View", + "verbose_name_plural": "Data Views", + }, + ), + migrations.CreateModel( + name="OsmData", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("xml", models.TextField()), + ("osm_id", models.CharField(max_length=20)), + ("tags", django.contrib.postgres.fields.jsonb.JSONField(default=dict)), + ( + "geom", + django.contrib.gis.db.models.fields.GeometryCollectionField( + srid=4326 + ), + ), + ("filename", models.CharField(max_length=255)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ("deleted_at", models.DateTimeField(default=None, null=True)), + ( + "instance", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="osm_data", + to="logger.instance", + ), + ), + ( + "field_name", + models.CharField(blank=True, default=b"", max_length=255), + ), + ("osm_type", models.CharField(default=b"way", max_length=10)), + ], + options={ + "unique_together": {("instance", "field_name")}, + }, + ), + migrations.CreateModel( + name="Widget", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("object_id", models.PositiveIntegerField()), + ( + "widget_type", + models.CharField( + choices=[(b"charts", b"Charts")], + default=b"charts", + max_length=25, + ), + ), + ("view_type", models.CharField(max_length=50)), + ("column", models.CharField(max_length=255)), + ( + "group_by", + models.CharField( + blank=True, default=None, max_length=255, null=True + ), + ), + ( + "title", + models.CharField( + blank=True, default=None, max_length=255, null=True + ), + ), + ( + "description", + models.CharField( + blank=True, default=None, max_length=255, null=True + ), + ), + ("key", models.CharField(db_index=True, max_length=32, unique=True)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "content_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="contenttypes.contenttype", + ), + ), + ( + "order", + models.PositiveIntegerField( + db_index=True, default=0, editable=False + ), + ), + ( + "aggregation", + models.CharField( + blank=True, default=None, max_length=255, null=True + ), + ), + ( + "metadata", + django.contrib.postgres.fields.jsonb.JSONField( + blank=True, default=dict + ), + ), + ], + options={ + "ordering": ("order",), + }, + ), + migrations.CreateModel( + name="OpenData", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=255)), + ( + "uuid", + models.CharField( + default=onadata.libs.utils.common_tools.get_uuid, + max_length=32, + unique=True, + ), + ), + ("object_id", models.PositiveIntegerField(blank=True, null=True)), + ("active", models.BooleanField(default=True)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "content_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="contenttypes.contenttype", + ), + ), + ], + ), + migrations.CreateModel( + name="Attachment", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "media_file", + models.FileField( + max_length=255, + upload_to=onadata.apps.logger.models.attachment.upload_to, + ), + ), + ("mimetype", models.CharField(blank=True, default=b"", max_length=100)), + ( + "extension", + models.CharField(db_index=True, default="non", max_length=10), + ), + ( + "instance", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="attachments", + to="logger.instance", + ), + ), + ("date_created", models.DateTimeField(auto_now_add=True, null=True)), + ("date_modified", models.DateTimeField(auto_now=True, null=True)), + ("deleted_at", models.DateTimeField(default=None, null=True)), + ("file_size", models.PositiveIntegerField(default=0)), + ], + options={ + "ordering": ("pk",), + }, + ), + migrations.CreateModel( + name="MergedXForm", + fields=[ + ( + "xform_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="logger.xform", + ), + ), + ( + "xforms", + models.ManyToManyField( + related_name="mergedxform_ptr", to="logger.XForm" + ), + ), + ], + options={ + "permissions": (("view_mergedxform", "Can view associated data"),), + }, + bases=("logger.xform",), + ), + migrations.CreateModel( + name="InstanceHistory", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("xml", models.TextField()), + ("uuid", models.CharField(default="", max_length=249)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "xform_instance", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="submission_history", + to="logger.instance", + ), + ), + ( + "geom", + django.contrib.gis.db.models.fields.GeometryCollectionField( + null=True, srid=4326 + ), + ), + ( + "user", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ("submission_date", models.DateTimeField(default=None, null=True)), + ("checksum", models.CharField(blank=True, max_length=64, null=True)), + ], + ), + migrations.RunSQL( + sql="UPDATE logger_xform SET hash = CONCAT('md5:', MD5(XML)) WHERE hash IS NULL;", + reverse_sql="", + ), + migrations.AddField( + model_name="attachment", + name="name", + field=models.CharField(blank=True, max_length=100, null=True), + ), + migrations.AlterField( + model_name="xform", + name="uuid", + field=models.CharField(default="", max_length=36), + ), + migrations.AddField( + model_name="dataview", + name="deleted_at", + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name="dataview", + name="deleted_by", + field=models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="dataview_deleted_by", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AddField( + model_name="xform", + name="deleted_by", + field=models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="xform_deleted_by", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AddField( + model_name="project", + name="deleted_by", + field=models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="project_deleted_by", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.RunPython(recalculate_xform_hash), + migrations.AddField( + model_name="instance", + name="deleted_by", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="deleted_instances", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AddField( + model_name="instance", + name="has_a_review", + field=models.BooleanField(default=False, verbose_name="has_a_review"), + ), + migrations.CreateModel( + name="SubmissionReview", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "status", + models.CharField( + choices=[ + ("1", "Approved"), + ("3", "Pending"), + ("2", "Rejected"), + ], + db_index=True, + default="3", + max_length=1, + verbose_name="Status", + ), + ), + ( + "deleted_at", + models.DateTimeField(db_index=True, default=None, null=True), + ), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "created_by", + models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "deleted_by", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="deleted_reviews", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "instance", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="reviews", + to="logger.instance", + ), + ), + ( + "note", + models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="notes", + to="logger.note", + ), + ), + ], + ), + migrations.AlterModelOptions( + name="mergedxform", + options={}, + ), + migrations.AlterModelOptions( + name="note", + options={}, + ), + migrations.AlterModelOptions( + name="project", + options={ + "permissions": ( + ("add_project_xform", "Can add xform to project"), + ("report_project_xform", "Can make submissions to the project"), + ("transfer_project", "Can transfer project to different owner"), + ("can_export_project_data", "Can export data in project"), + ("view_project_all", "Can view all associated data"), + ("view_project_data", "Can view submitted data"), + ) + }, + ), + migrations.AlterModelOptions( + name="xform", + options={ + "ordering": ("pk",), + "permissions": ( + ("view_xform_all", "Can view all associated data"), + ("view_xform_data", "Can view submitted data"), + ("report_xform", "Can make submissions to the form"), + ("move_xform", "Can move form between projects"), + ("transfer_xform", "Can transfer form ownership."), + ("can_export_xform_data", "Can export form data"), + ("delete_submission", "Can delete submissions from form"), + ), + "verbose_name": "XForm", + "verbose_name_plural": "XForms", + }, + ), + migrations.AlterField( + model_name="attachment", + name="mimetype", + field=models.CharField(blank=True, default="", max_length=100), + ), + migrations.AlterField( + model_name="instance", + name="survey_type", + field=models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, to="logger.surveytype" + ), + ), + migrations.AlterField( + model_name="instance", + name="user", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="instances", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AlterField( + model_name="osmdata", + name="field_name", + field=models.CharField(blank=True, default="", max_length=255), + ), + migrations.AlterField( + model_name="osmdata", + name="osm_type", + field=models.CharField(default="way", max_length=10), + ), + migrations.AlterField( + model_name="project", + name="tags", + field=taggit.managers.TaggableManager( + help_text="A comma-separated list of tags.", + related_name="project_tags", + through="taggit.TaggedItem", + to="taggit.Tag", + verbose_name="Tags", + ), + ), + migrations.AlterField( + model_name="widget", + name="order", + field=models.PositiveIntegerField( + db_index=True, editable=False, verbose_name="order" + ), + ), + migrations.AlterField( + model_name="widget", + name="widget_type", + field=models.CharField( + choices=[("charts", "Charts")], default="charts", max_length=25 + ), + ), + migrations.AlterField( + model_name="xform", + name="created_by", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AlterField( + model_name="xform", + name="sms_id_string", + field=models.SlugField( + default="", editable=False, max_length=100, verbose_name="SMS ID" + ), + ), + migrations.AddField( + model_name="xform", + name="public_key", + field=models.TextField(blank=True, default="", null=True), + ), + migrations.AddField( + model_name="attachment", + name="deleted_by", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="deleted_attachments", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AlterField( + model_name="xform", + name="uuid", + field=models.CharField(db_index=True, default="", max_length=36), + ), + migrations.RunPython(generate_uuid_if_missing), + migrations.RunPython(regenerate_instance_json), + migrations.CreateModel( + name="XFormVersion", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("xls", models.FileField(upload_to="")), + ("version", models.CharField(max_length=100)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ("xml", models.TextField()), + ("json", models.TextField()), + ( + "created_by", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "xform", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="versions", + to="logger.xform", + ), + ), + ], + options={ + "unique_together": {("xform", "version")}, + }, + ), + migrations.RunPython(create_initial_xform_version), + ] diff --git a/onadata/apps/logger/migrations/0002_auto_20150717_0048.py b/onadata/apps/logger/migrations/0002_auto_20150717_0048.py index 4ae94dca71..6293c1ed9a 100644 --- a/onadata/apps/logger/migrations/0002_auto_20150717_0048.py +++ b/onadata/apps/logger/migrations/0002_auto_20150717_0048.py @@ -7,25 +7,25 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0001_initial'), + ("logger", "0001_initial"), ] operations = [ migrations.AddField( - model_name='attachment', - name='date_created', + model_name="attachment", + name="date_created", field=models.DateTimeField(auto_now_add=True, null=True), preserve_default=True, ), migrations.AddField( - model_name='attachment', - name='date_modified', + model_name="attachment", + name="date_modified", field=models.DateTimeField(auto_now=True, null=True), preserve_default=True, ), migrations.AddField( - model_name='attachment', - name='deleted_at', + model_name="attachment", + name="deleted_at", field=models.DateTimeField(default=None, null=True), preserve_default=True, ), diff --git a/onadata/apps/logger/migrations/0002_auto_20220425_0340.py b/onadata/apps/logger/migrations/0002_auto_20220425_0340.py new file mode 100644 index 0000000000..90d13c6afe --- /dev/null +++ b/onadata/apps/logger/migrations/0002_auto_20220425_0340.py @@ -0,0 +1,63 @@ +# Generated by Django 3.2.13 on 2022-04-25 07:40 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("logger", "0001_pre-django-3-upgrade"), + ] + + operations = [ + migrations.AlterField( + model_name="dataview", + name="columns", + field=models.JSONField(), + ), + migrations.AlterField( + model_name="dataview", + name="query", + field=models.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name="instance", + name="json", + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name="instance", + name="xform", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="instances", + to="logger.xform", + ), + ), + migrations.AlterField( + model_name="osmdata", + name="tags", + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name="project", + name="metadata", + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name="widget", + name="metadata", + field=models.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name="xform", + name="json", + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name="xform", + name="last_updated_at", + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/onadata/apps/logger/migrations/0003_alter_instance_media_all_received.py b/onadata/apps/logger/migrations/0003_alter_instance_media_all_received.py new file mode 100644 index 0000000000..b4fadc7c4a --- /dev/null +++ b/onadata/apps/logger/migrations/0003_alter_instance_media_all_received.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2.13 on 2022-04-25 08:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("logger", "0002_auto_20220425_0340"), + ] + + operations = [ + migrations.AlterField( + model_name="instance", + name="media_all_received", + field=models.BooleanField( + default=True, null=True, verbose_name="Received All Media Attachemts" + ), + ), + ] diff --git a/onadata/apps/logger/migrations/0003_dataview_instances_with_geopoints.py b/onadata/apps/logger/migrations/0003_dataview_instances_with_geopoints.py index af36fdd965..3387d4ccf0 100644 --- a/onadata/apps/logger/migrations/0003_dataview_instances_with_geopoints.py +++ b/onadata/apps/logger/migrations/0003_dataview_instances_with_geopoints.py @@ -7,13 +7,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0002_auto_20150717_0048'), + ("logger", "0002_auto_20150717_0048"), ] operations = [ migrations.AddField( - model_name='dataview', - name='instances_with_geopoints', + model_name="dataview", + name="instances_with_geopoints", field=models.BooleanField(default=False), preserve_default=True, ), diff --git a/onadata/apps/logger/migrations/0004_auto_20150910_0056.py b/onadata/apps/logger/migrations/0004_auto_20150910_0056.py index 5949179146..86e737c21a 100644 --- a/onadata/apps/logger/migrations/0004_auto_20150910_0056.py +++ b/onadata/apps/logger/migrations/0004_auto_20150910_0056.py @@ -8,52 +8,77 @@ class Migration(migrations.Migration): dependencies = [ - ('auth', '0001_initial'), + ("auth", "0001_initial"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0003_dataview_instances_with_geopoints'), + ("logger", "0003_dataview_instances_with_geopoints"), ] operations = [ migrations.CreateModel( - name='ProjectGroupObjectPermission', + name="ProjectGroupObjectPermission", fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('content_object', models.ForeignKey( - to='logger.Project', on_delete=models.CASCADE)), - ('group', models.ForeignKey(to='auth.Group', - on_delete=models.CASCADE)), - ('permission', models.ForeignKey(to='auth.Permission', - on_delete=models.CASCADE)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ( + "content_object", + models.ForeignKey(to="logger.Project", on_delete=models.CASCADE), + ), + ("group", models.ForeignKey(to="auth.Group", on_delete=models.CASCADE)), + ( + "permission", + models.ForeignKey(to="auth.Permission", on_delete=models.CASCADE), + ), ], options={ - 'abstract': False, + "abstract": False, }, bases=(models.Model,), ), migrations.CreateModel( - name='ProjectUserObjectPermission', + name="ProjectUserObjectPermission", fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('content_object', models.ForeignKey(to='logger.Project', - on_delete=models.CASCADE)), - ('permission', models.ForeignKey(to='auth.Permission', - on_delete=models.CASCADE)), - ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, - on_delete=models.CASCADE)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ( + "content_object", + models.ForeignKey(to="logger.Project", on_delete=models.CASCADE), + ), + ( + "permission", + models.ForeignKey(to="auth.Permission", on_delete=models.CASCADE), + ), + ( + "user", + models.ForeignKey( + to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE + ), + ), ], options={ - 'abstract': False, + "abstract": False, }, bases=(models.Model,), ), migrations.AlterUniqueTogether( - name='projectuserobjectpermission', - unique_together=set([('user', 'permission', 'content_object')]), + name="projectuserobjectpermission", + unique_together=set([("user", "permission", "content_object")]), ), migrations.AlterUniqueTogether( - name='projectgroupobjectpermission', - unique_together=set([('group', 'permission', 'content_object')]), + name="projectgroupobjectpermission", + unique_together=set([("group", "permission", "content_object")]), ), ] diff --git a/onadata/apps/logger/migrations/0005_auto_20151015_0758.py b/onadata/apps/logger/migrations/0005_auto_20151015_0758.py index db63190a51..4ea296e7e5 100644 --- a/onadata/apps/logger/migrations/0005_auto_20151015_0758.py +++ b/onadata/apps/logger/migrations/0005_auto_20151015_0758.py @@ -8,52 +8,77 @@ class Migration(migrations.Migration): dependencies = [ - ('auth', '0001_initial'), + ("auth", "0001_initial"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0004_auto_20150910_0056'), + ("logger", "0004_auto_20150910_0056"), ] operations = [ migrations.CreateModel( - name='XFormGroupObjectPermission', + name="XFormGroupObjectPermission", fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('content_object', models.ForeignKey(to='logger.XForm', - on_delete=models.CASCADE)), - ('group', models.ForeignKey(to='auth.Group', - on_delete=models.CASCADE)), - ('permission', models.ForeignKey(to='auth.Permission', - on_delete=models.CASCADE)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ( + "content_object", + models.ForeignKey(to="logger.XForm", on_delete=models.CASCADE), + ), + ("group", models.ForeignKey(to="auth.Group", on_delete=models.CASCADE)), + ( + "permission", + models.ForeignKey(to="auth.Permission", on_delete=models.CASCADE), + ), ], options={ - 'abstract': False, + "abstract": False, }, bases=(models.Model,), ), migrations.CreateModel( - name='XFormUserObjectPermission', + name="XFormUserObjectPermission", fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('content_object', models.ForeignKey(to='logger.XForm', - on_delete=models.CASCADE)), - ('permission', models.ForeignKey(to='auth.Permission', - on_delete=models.CASCADE)), - ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, - on_delete=models.CASCADE)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ( + "content_object", + models.ForeignKey(to="logger.XForm", on_delete=models.CASCADE), + ), + ( + "permission", + models.ForeignKey(to="auth.Permission", on_delete=models.CASCADE), + ), + ( + "user", + models.ForeignKey( + to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE + ), + ), ], options={ - 'abstract': False, + "abstract": False, }, bases=(models.Model,), ), migrations.AlterUniqueTogether( - name='xformuserobjectpermission', - unique_together=set([('user', 'permission', 'content_object')]), + name="xformuserobjectpermission", + unique_together=set([("user", "permission", "content_object")]), ), migrations.AlterUniqueTogether( - name='xformgroupobjectpermission', - unique_together=set([('group', 'permission', 'content_object')]), + name="xformgroupobjectpermission", + unique_together=set([("group", "permission", "content_object")]), ), ] diff --git a/onadata/apps/logger/migrations/0006_auto_20151106_0130.py b/onadata/apps/logger/migrations/0006_auto_20151106_0130.py index aa0f1b0137..c29b565ef7 100644 --- a/onadata/apps/logger/migrations/0006_auto_20151106_0130.py +++ b/onadata/apps/logger/migrations/0006_auto_20151106_0130.py @@ -2,52 +2,67 @@ from __future__ import unicode_literals from django.db import models, migrations -import jsonfield.fields import django.contrib.gis.db.models.fields class Migration(migrations.Migration): dependencies = [ - ('logger', '0005_auto_20151015_0758'), + ("logger", "0005_auto_20151015_0758"), ] operations = [ migrations.CreateModel( - name='OsmData', + name="OsmData", fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('xml', models.TextField()), - ('osm_id', models.CharField(max_length=10)), - ('tags', jsonfield.fields.JSONField(default={})), - ('geom', - django.contrib.gis.db.models.fields.GeometryCollectionField( - srid=4326)), - ('filename', models.CharField(max_length=255)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('deleted_at', models.DateTimeField(default=None, null=True)), - ('instance', models.ForeignKey(related_name='osm_data', - to='logger.Instance', - on_delete=models.CASCADE)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("xml", models.TextField()), + ("osm_id", models.CharField(max_length=10)), + ("tags", models.JSONField(default={})), + ( + "geom", + django.contrib.gis.db.models.fields.GeometryCollectionField( + srid=4326 + ), + ), + ("filename", models.CharField(max_length=255)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ("deleted_at", models.DateTimeField(default=None, null=True)), + ( + "instance", + models.ForeignKey( + related_name="osm_data", + to="logger.Instance", + on_delete=models.CASCADE, + ), + ), ], - options={ - }, + options={}, bases=(models.Model,), ), migrations.AlterModelOptions( - name='xform', + name="xform", options={ - 'ordering': ('id_string',), 'verbose_name': 'XForm', - 'verbose_name_plural': 'XForms', - 'permissions': ( - ('view_xform', 'Can view associated data'), - ('report_xform', 'Can make submissions to the form'), - ('move_xform', 'Can move form between projects'), - ('transfer_xform', 'Can transfer form ownership.'), - ('can_export_xform_data', 'Can export form data'), - ('delete_submission', 'Can delete submissions from form') - )}, + "ordering": ("id_string",), + "verbose_name": "XForm", + "verbose_name_plural": "XForms", + "permissions": ( + ("view_xform", "Can view associated data"), + ("report_xform", "Can make submissions to the form"), + ("move_xform", "Can move form between projects"), + ("transfer_xform", "Can transfer form ownership."), + ("can_export_xform_data", "Can export form data"), + ("delete_submission", "Can delete submissions from form"), + ), + }, ), ] diff --git a/onadata/apps/logger/migrations/0007_osmdata_field_name.py b/onadata/apps/logger/migrations/0007_osmdata_field_name.py index 7870df2c4e..5d579a1110 100644 --- a/onadata/apps/logger/migrations/0007_osmdata_field_name.py +++ b/onadata/apps/logger/migrations/0007_osmdata_field_name.py @@ -7,14 +7,14 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0006_auto_20151106_0130'), + ("logger", "0006_auto_20151106_0130"), ] operations = [ migrations.AddField( - model_name='osmdata', - name='field_name', - field=models.CharField(default=b'', max_length=255, blank=True), + model_name="osmdata", + name="field_name", + field=models.CharField(default=b"", max_length=255, blank=True), preserve_default=True, ), ] diff --git a/onadata/apps/logger/migrations/0008_osmdata_osm_type.py b/onadata/apps/logger/migrations/0008_osmdata_osm_type.py index 7bde859ecf..e0f3354ea1 100644 --- a/onadata/apps/logger/migrations/0008_osmdata_osm_type.py +++ b/onadata/apps/logger/migrations/0008_osmdata_osm_type.py @@ -7,14 +7,14 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0007_osmdata_field_name'), + ("logger", "0007_osmdata_field_name"), ] operations = [ migrations.AddField( - model_name='osmdata', - name='osm_type', - field=models.CharField(default=b'way', max_length=10), + model_name="osmdata", + name="osm_type", + field=models.CharField(default=b"way", max_length=10), preserve_default=True, ), ] diff --git a/onadata/apps/logger/migrations/0009_auto_20151111_0438.py b/onadata/apps/logger/migrations/0009_auto_20151111_0438.py index 96113b0393..2da2ae7381 100644 --- a/onadata/apps/logger/migrations/0009_auto_20151111_0438.py +++ b/onadata/apps/logger/migrations/0009_auto_20151111_0438.py @@ -7,12 +7,12 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0008_osmdata_osm_type'), + ("logger", "0008_osmdata_osm_type"), ] operations = [ migrations.AlterUniqueTogether( - name='osmdata', - unique_together=set([('instance', 'field_name')]), + name="osmdata", + unique_together=set([("instance", "field_name")]), ), ] diff --git a/onadata/apps/logger/migrations/0010_attachment_file_size.py b/onadata/apps/logger/migrations/0010_attachment_file_size.py index 63bcc1c617..71224e848c 100644 --- a/onadata/apps/logger/migrations/0010_attachment_file_size.py +++ b/onadata/apps/logger/migrations/0010_attachment_file_size.py @@ -7,13 +7,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0009_auto_20151111_0438'), + ("logger", "0009_auto_20151111_0438"), ] operations = [ migrations.AddField( - model_name='attachment', - name='file_size', + model_name="attachment", + name="file_size", field=models.PositiveIntegerField(default=0), preserve_default=True, ), diff --git a/onadata/apps/logger/migrations/0011_dataview_matches_parent.py b/onadata/apps/logger/migrations/0011_dataview_matches_parent.py index cee1b46ea5..14e0742af8 100644 --- a/onadata/apps/logger/migrations/0011_dataview_matches_parent.py +++ b/onadata/apps/logger/migrations/0011_dataview_matches_parent.py @@ -7,13 +7,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0010_attachment_file_size'), + ("logger", "0010_attachment_file_size"), ] operations = [ migrations.AddField( - model_name='dataview', - name='matches_parent', + model_name="dataview", + name="matches_parent", field=models.BooleanField(default=False), preserve_default=True, ), diff --git a/onadata/apps/logger/migrations/0012_auto_20160114_0708.py b/onadata/apps/logger/migrations/0012_auto_20160114_0708.py index ccbae70aaa..c82f2b402c 100644 --- a/onadata/apps/logger/migrations/0012_auto_20160114_0708.py +++ b/onadata/apps/logger/migrations/0012_auto_20160114_0708.py @@ -7,14 +7,14 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0011_dataview_matches_parent'), + ("logger", "0011_dataview_matches_parent"), ] operations = [ migrations.AlterField( - model_name='attachment', - name='mimetype', - field=models.CharField(default=b'', max_length=100, blank=True), + model_name="attachment", + name="mimetype", + field=models.CharField(default=b"", max_length=100, blank=True), preserve_default=True, ), ] diff --git a/onadata/apps/logger/migrations/0013_note_created_by.py b/onadata/apps/logger/migrations/0013_note_created_by.py index eae6d40b75..8fbc71ff40 100644 --- a/onadata/apps/logger/migrations/0013_note_created_by.py +++ b/onadata/apps/logger/migrations/0013_note_created_by.py @@ -9,15 +9,19 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0012_auto_20160114_0708'), + ("logger", "0012_auto_20160114_0708"), ] operations = [ migrations.AddField( - model_name='note', - name='created_by', - field=models.ForeignKey(blank=True, to=settings.AUTH_USER_MODEL, - null=True, on_delete=models.CASCADE), + model_name="note", + name="created_by", + field=models.ForeignKey( + blank=True, + to=settings.AUTH_USER_MODEL, + null=True, + on_delete=models.CASCADE, + ), preserve_default=True, ), ] diff --git a/onadata/apps/logger/migrations/0014_note_instance_field.py b/onadata/apps/logger/migrations/0014_note_instance_field.py index 8db30ebdae..da25310973 100644 --- a/onadata/apps/logger/migrations/0014_note_instance_field.py +++ b/onadata/apps/logger/migrations/0014_note_instance_field.py @@ -7,13 +7,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0013_note_created_by'), + ("logger", "0013_note_created_by"), ] operations = [ migrations.AddField( - model_name='note', - name='instance_field', + model_name="note", + name="instance_field", field=models.TextField(null=True, blank=True), preserve_default=True, ), diff --git a/onadata/apps/logger/migrations/0015_auto_20160222_0559.py b/onadata/apps/logger/migrations/0015_auto_20160222_0559.py index eb6cbcccd4..212a30a7e3 100644 --- a/onadata/apps/logger/migrations/0015_auto_20160222_0559.py +++ b/onadata/apps/logger/migrations/0015_auto_20160222_0559.py @@ -7,19 +7,18 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0014_note_instance_field'), + ("logger", "0014_note_instance_field"), ] operations = [ migrations.AlterModelOptions( - name='widget', - options={'ordering': ('order',)}, + name="widget", + options={"ordering": ("order",)}, ), migrations.AddField( - model_name='widget', - name='order', - field=models.PositiveIntegerField(default=0, editable=False, - db_index=True), + model_name="widget", + name="order", + field=models.PositiveIntegerField(default=0, editable=False, db_index=True), preserve_default=False, ), ] diff --git a/onadata/apps/logger/migrations/0016_widget_aggregation.py b/onadata/apps/logger/migrations/0016_widget_aggregation.py index d429a246ea..3f6aa8bda3 100644 --- a/onadata/apps/logger/migrations/0016_widget_aggregation.py +++ b/onadata/apps/logger/migrations/0016_widget_aggregation.py @@ -7,15 +7,14 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0015_auto_20160222_0559'), + ("logger", "0015_auto_20160222_0559"), ] operations = [ migrations.AddField( - model_name='widget', - name='aggregation', - field=models.CharField(default=None, max_length=255, null=True, - blank=True), + model_name="widget", + name="aggregation", + field=models.CharField(default=None, max_length=255, null=True, blank=True), preserve_default=True, ), ] diff --git a/onadata/apps/logger/migrations/0017_auto_20160224_0130.py b/onadata/apps/logger/migrations/0017_auto_20160224_0130.py index f6e71f7031..9d4fd9d38e 100644 --- a/onadata/apps/logger/migrations/0017_auto_20160224_0130.py +++ b/onadata/apps/logger/migrations/0017_auto_20160224_0130.py @@ -7,26 +7,29 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0016_widget_aggregation'), + ("logger", "0016_widget_aggregation"), ] operations = [ migrations.AlterField( - model_name='instance', - name='uuid', + model_name="instance", + name="uuid", field=models.CharField(max_length=249), preserve_default=True, ), migrations.AlterField( - model_name='instance', - name='xform', - field=models.ForeignKey(related_name='instances', default=-1, - to='logger.XForm', - on_delete=models.CASCADE), + model_name="instance", + name="xform", + field=models.ForeignKey( + related_name="instances", + default=-1, + to="logger.XForm", + on_delete=models.CASCADE, + ), preserve_default=False, ), migrations.AlterUniqueTogether( - name='instance', - unique_together=set([('xform', 'uuid')]), + name="instance", + unique_together=set([("xform", "uuid")]), ), ] diff --git a/onadata/apps/logger/migrations/0018_auto_20160301_0330.py b/onadata/apps/logger/migrations/0018_auto_20160301_0330.py index 077a7dce24..bde2f67839 100644 --- a/onadata/apps/logger/migrations/0018_auto_20160301_0330.py +++ b/onadata/apps/logger/migrations/0018_auto_20160301_0330.py @@ -10,23 +10,24 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0017_auto_20160224_0130'), + ("logger", "0017_auto_20160224_0130"), ] operations = [ migrations.AddField( - model_name='instancehistory', - name='geom', + model_name="instancehistory", + name="geom", field=django.contrib.gis.db.models.fields.GeometryCollectionField( - srid=4326, null=True), + srid=4326, null=True + ), preserve_default=True, ), migrations.AddField( - model_name='instancehistory', - name='user', + model_name="instancehistory", + name="user", field=models.ForeignKey( - to=settings.AUTH_USER_MODEL, null=True, - on_delete=models.CASCADE), + to=settings.AUTH_USER_MODEL, null=True, on_delete=models.CASCADE + ), preserve_default=True, ), ] diff --git a/onadata/apps/logger/migrations/0019_auto_20160307_0256.py b/onadata/apps/logger/migrations/0019_auto_20160307_0256.py index bbb5f3bdfd..09c2af5b19 100644 --- a/onadata/apps/logger/migrations/0019_auto_20160307_0256.py +++ b/onadata/apps/logger/migrations/0019_auto_20160307_0256.py @@ -2,34 +2,36 @@ from __future__ import unicode_literals from django.db import models, migrations -import jsonfield.fields class Migration(migrations.Migration): dependencies = [ - ('logger', '0018_auto_20160301_0330'), + ("logger", "0018_auto_20160301_0330"), ] operations = [ migrations.AddField( - model_name='widget', - name='metadata', - field=jsonfield.fields.JSONField(default={}, blank=True), + model_name="widget", + name="metadata", + field=models.JSONField(default={}, blank=True), preserve_default=True, ), migrations.AlterField( - model_name='instance', - name='uuid', - field=models.CharField(default='', max_length=249), + model_name="instance", + name="uuid", + field=models.CharField(default="", max_length=249), preserve_default=True, ), migrations.AlterField( - model_name='instance', - name='xform', - field=models.ForeignKey(related_name='instances', - to='logger.XForm', null=True, - on_delete=models.CASCADE), + model_name="instance", + name="xform", + field=models.ForeignKey( + related_name="instances", + to="logger.XForm", + null=True, + on_delete=models.CASCADE, + ), preserve_default=True, ), ] diff --git a/onadata/apps/logger/migrations/0020_auto_20160408_0325.py b/onadata/apps/logger/migrations/0020_auto_20160408_0325.py index 9353329eae..141ec5f544 100644 --- a/onadata/apps/logger/migrations/0020_auto_20160408_0325.py +++ b/onadata/apps/logger/migrations/0020_auto_20160408_0325.py @@ -9,40 +9,42 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0019_auto_20160307_0256'), + ("logger", "0019_auto_20160307_0256"), ] operations = [ migrations.AlterField( - model_name='dataview', - name='columns', + model_name="dataview", + name="columns", field=django.contrib.postgres.fields.jsonb.JSONField(), ), migrations.AlterField( - model_name='dataview', - name='query', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, - default=dict), + model_name="dataview", + name="query", + field=django.contrib.postgres.fields.jsonb.JSONField( + blank=True, default=dict + ), ), migrations.AlterField( - model_name='instance', - name='json', + model_name="instance", + name="json", field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), ), migrations.AlterField( - model_name='osmdata', - name='tags', + model_name="osmdata", + name="tags", field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), ), migrations.AlterField( - model_name='project', - name='metadata', + model_name="project", + name="metadata", field=django.contrib.postgres.fields.jsonb.JSONField(blank=True), ), migrations.AlterField( - model_name='widget', - name='metadata', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, - default=dict), + model_name="widget", + name="metadata", + field=django.contrib.postgres.fields.jsonb.JSONField( + blank=True, default=dict + ), ), ] diff --git a/onadata/apps/logger/migrations/0021_auto_20160408_0919.py b/onadata/apps/logger/migrations/0021_auto_20160408_0919.py index dbe27f3f87..56586ad9ae 100644 --- a/onadata/apps/logger/migrations/0021_auto_20160408_0919.py +++ b/onadata/apps/logger/migrations/0021_auto_20160408_0919.py @@ -9,13 +9,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0020_auto_20160408_0325'), + ("logger", "0020_auto_20160408_0325"), ] operations = [ migrations.AlterField( - model_name='project', - name='metadata', + model_name="project", + name="metadata", field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), ), ] diff --git a/onadata/apps/logger/migrations/0022_auto_20160418_0518.py b/onadata/apps/logger/migrations/0022_auto_20160418_0518.py index 8954428f43..aa92c8bfde 100644 --- a/onadata/apps/logger/migrations/0022_auto_20160418_0518.py +++ b/onadata/apps/logger/migrations/0022_auto_20160418_0518.py @@ -8,18 +8,18 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0021_auto_20160408_0919'), + ("logger", "0021_auto_20160408_0919"), ] operations = [ migrations.AddField( - model_name='instance', - name='last_edited', + model_name="instance", + name="last_edited", field=models.DateTimeField(default=None, null=True), ), migrations.AddField( - model_name='instancehistory', - name='submission_date', + model_name="instancehistory", + name="submission_date", field=models.DateTimeField(default=None, null=True), ), ] diff --git a/onadata/apps/logger/migrations/0023_auto_20160419_0403.py b/onadata/apps/logger/migrations/0023_auto_20160419_0403.py index dc99e5bdb0..810289c740 100644 --- a/onadata/apps/logger/migrations/0023_auto_20160419_0403.py +++ b/onadata/apps/logger/migrations/0023_auto_20160419_0403.py @@ -8,19 +8,18 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0022_auto_20160418_0518'), + ("logger", "0022_auto_20160418_0518"), ] operations = [ migrations.AlterField( - model_name='widget', - name='column', + model_name="widget", + name="column", field=models.CharField(max_length=255), ), migrations.AlterField( - model_name='widget', - name='group_by', - field=models.CharField(blank=True, default=None, max_length=255, - null=True), + model_name="widget", + name="group_by", + field=models.CharField(blank=True, default=None, max_length=255, null=True), ), ] diff --git a/onadata/apps/logger/migrations/0024_xform_has_hxl_support.py b/onadata/apps/logger/migrations/0024_xform_has_hxl_support.py index 64a4ad8912..acecce134f 100644 --- a/onadata/apps/logger/migrations/0024_xform_has_hxl_support.py +++ b/onadata/apps/logger/migrations/0024_xform_has_hxl_support.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0023_auto_20160419_0403'), + ("logger", "0023_auto_20160419_0403"), ] operations = [ migrations.AddField( - model_name='xform', - name='has_hxl_support', + model_name="xform", + name="has_hxl_support", field=models.BooleanField(default=False), ), ] diff --git a/onadata/apps/logger/migrations/0025_xform_last_updated_at.py b/onadata/apps/logger/migrations/0025_xform_last_updated_at.py index f8000e7012..9d1ec9a4f9 100644 --- a/onadata/apps/logger/migrations/0025_xform_last_updated_at.py +++ b/onadata/apps/logger/migrations/0025_xform_last_updated_at.py @@ -10,17 +10,17 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0024_xform_has_hxl_support'), + ("logger", "0024_xform_has_hxl_support"), ] operations = [ migrations.AddField( - model_name='xform', - name='last_updated_at', + model_name="xform", + name="last_updated_at", field=models.DateTimeField( auto_now=True, - default=datetime.datetime(2016, 8, 18, 12, 43, 30, 316792, - tzinfo=utc)), + default=datetime.datetime(2016, 8, 18, 12, 43, 30, 316792, tzinfo=utc), + ), preserve_default=False, ), ] diff --git a/onadata/apps/logger/migrations/0026_auto_20160913_0239.py b/onadata/apps/logger/migrations/0026_auto_20160913_0239.py index d29b823400..173122a4a5 100644 --- a/onadata/apps/logger/migrations/0026_auto_20160913_0239.py +++ b/onadata/apps/logger/migrations/0026_auto_20160913_0239.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0025_xform_last_updated_at'), + ("logger", "0025_xform_last_updated_at"), ] operations = [ migrations.AlterField( - model_name='osmdata', - name='osm_id', + model_name="osmdata", + name="osm_id", field=models.CharField(max_length=20), ), ] diff --git a/onadata/apps/logger/migrations/0027_auto_20161201_0730.py b/onadata/apps/logger/migrations/0027_auto_20161201_0730.py index d253875e2c..559950558d 100644 --- a/onadata/apps/logger/migrations/0027_auto_20161201_0730.py +++ b/onadata/apps/logger/migrations/0027_auto_20161201_0730.py @@ -8,14 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0026_auto_20160913_0239'), + ("logger", "0026_auto_20160913_0239"), ] operations = [ migrations.AlterField( - model_name='widget', - name='title', - field=models.CharField(blank=True, default=None, max_length=255, - null=True), + model_name="widget", + name="title", + field=models.CharField(blank=True, default=None, max_length=255, null=True), ), ] diff --git a/onadata/apps/logger/migrations/0028_auto_20170217_0502.py b/onadata/apps/logger/migrations/0028_auto_20170217_0502.py index 39734a6f06..fbb3dcbfa3 100644 --- a/onadata/apps/logger/migrations/0028_auto_20170217_0502.py +++ b/onadata/apps/logger/migrations/0028_auto_20170217_0502.py @@ -9,44 +9,51 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0027_auto_20161201_0730'), + ("logger", "0027_auto_20161201_0730"), ] operations = [ migrations.AlterModelOptions( - name='project', - options={'permissions': ( - ('view_project', 'Can view project'), - ('add_project_xform', 'Can add xform to project'), - ('report_project_xform', - 'Can make submissions to the project'), - ('transfer_project', - 'Can transfer project to different owner'), - ('can_export_project_data', 'Can export data in project'), - ('view_project_all', 'Can view all associated data'), - ('view_project_data', 'Can view submitted data'))}, + name="project", + options={ + "permissions": ( + ("view_project", "Can view project"), + ("add_project_xform", "Can add xform to project"), + ("report_project_xform", "Can make submissions to the project"), + ("transfer_project", "Can transfer project to different owner"), + ("can_export_project_data", "Can export data in project"), + ("view_project_all", "Can view all associated data"), + ("view_project_data", "Can view submitted data"), + ) + }, ), migrations.AlterModelOptions( - name='xform', + name="xform", options={ - 'ordering': ('id_string',), - 'permissions': ( - ('view_xform', 'Can view associated data'), - ('view_xform_all', 'Can view all associated data'), - ('view_xform_data', 'Can view submitted data'), - ('report_xform', 'Can make submissions to the form'), - ('move_xform', 'Can move form between projects'), - ('transfer_xform', 'Can transfer form ownership.'), - ('can_export_xform_data', 'Can export form data'), - ('delete_submission', 'Can delete submissions from form')), - 'verbose_name': 'XForm', 'verbose_name_plural': 'XForms'}, + "ordering": ("id_string",), + "permissions": ( + ("view_xform", "Can view associated data"), + ("view_xform_all", "Can view all associated data"), + ("view_xform_data", "Can view submitted data"), + ("report_xform", "Can make submissions to the form"), + ("move_xform", "Can move form between projects"), + ("transfer_xform", "Can transfer form ownership."), + ("can_export_xform_data", "Can export form data"), + ("delete_submission", "Can delete submissions from form"), + ), + "verbose_name": "XForm", + "verbose_name_plural": "XForms", + }, ), migrations.AlterField( - model_name='instance', - name='xform', + model_name="instance", + name="xform", field=models.ForeignKey( - default=328, on_delete=django.db.models.deletion.CASCADE, - related_name='instances', to='logger.XForm'), + default=328, + on_delete=django.db.models.deletion.CASCADE, + related_name="instances", + to="logger.XForm", + ), preserve_default=False, ), ] diff --git a/onadata/apps/logger/migrations/0028_auto_20170221_0838.py b/onadata/apps/logger/migrations/0028_auto_20170221_0838.py index 07c68426c3..b059d64e03 100644 --- a/onadata/apps/logger/migrations/0028_auto_20170221_0838.py +++ b/onadata/apps/logger/migrations/0028_auto_20170221_0838.py @@ -9,26 +9,36 @@ class Migration(migrations.Migration): dependencies = [ - ('contenttypes', '0002_remove_content_type_name'), - ('logger', '0027_auto_20161201_0730'), + ("contenttypes", "0002_remove_content_type_name"), + ("logger", "0027_auto_20161201_0730"), ] operations = [ migrations.CreateModel( - name='OpenData', + name="OpenData", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, - serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=255)), - ('uuid', models.CharField(default='', max_length=32)), - ('object_id', models.PositiveIntegerField(blank=True, - null=True)), - ('active', models.BooleanField(default=True)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('content_type', models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to='contenttypes.ContentType')), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=255)), + ("uuid", models.CharField(default="", max_length=32)), + ("object_id", models.PositiveIntegerField(blank=True, null=True)), + ("active", models.BooleanField(default=True)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "content_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="contenttypes.ContentType", + ), + ), ], ), ] diff --git a/onadata/apps/logger/migrations/0029_auto_20170221_0908.py b/onadata/apps/logger/migrations/0029_auto_20170221_0908.py index 681b1890e9..c80adaf57b 100644 --- a/onadata/apps/logger/migrations/0029_auto_20170221_0908.py +++ b/onadata/apps/logger/migrations/0029_auto_20170221_0908.py @@ -9,14 +9,15 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0028_auto_20170221_0838'), + ("logger", "0028_auto_20170221_0838"), ] operations = [ migrations.AlterField( - model_name='opendata', - name='uuid', + model_name="opendata", + name="uuid", field=models.CharField( - default=onadata.apps.logger.models.open_data.get_uuid, - max_length=32), ), + default=onadata.apps.logger.models.open_data.get_uuid, max_length=32 + ), + ), ] diff --git a/onadata/apps/logger/migrations/0030_auto_20170227_0137.py b/onadata/apps/logger/migrations/0030_auto_20170227_0137.py index a4e879564f..a764d7efab 100644 --- a/onadata/apps/logger/migrations/0030_auto_20170227_0137.py +++ b/onadata/apps/logger/migrations/0030_auto_20170227_0137.py @@ -9,15 +9,17 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0029_auto_20170221_0908'), + ("logger", "0029_auto_20170221_0908"), ] operations = [ migrations.AlterField( - model_name='opendata', - name='uuid', + model_name="opendata", + name="uuid", field=models.CharField( default=onadata.libs.utils.common_tools.get_uuid, max_length=32, - unique=True), ), + unique=True, + ), + ), ] diff --git a/onadata/apps/logger/migrations/0031_merge.py b/onadata/apps/logger/migrations/0031_merge.py index 3d24ace1f5..4a1ebd5bc6 100644 --- a/onadata/apps/logger/migrations/0031_merge.py +++ b/onadata/apps/logger/migrations/0031_merge.py @@ -8,9 +8,8 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0028_auto_20170217_0502'), - ('logger', '0030_auto_20170227_0137'), + ("logger", "0028_auto_20170217_0502"), + ("logger", "0030_auto_20170227_0137"), ] - operations = [ - ] + operations = [] diff --git a/onadata/apps/logger/migrations/0032_project_deleted_at.py b/onadata/apps/logger/migrations/0032_project_deleted_at.py index 2fff7faf34..596027fdc2 100644 --- a/onadata/apps/logger/migrations/0032_project_deleted_at.py +++ b/onadata/apps/logger/migrations/0032_project_deleted_at.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0031_merge'), + ("logger", "0031_merge"), ] operations = [ migrations.AddField( - model_name='project', - name='deleted_at', + model_name="project", + name="deleted_at", field=models.DateTimeField(blank=True, null=True), ), ] diff --git a/onadata/apps/logger/migrations/0033_auto_20170705_0159.py b/onadata/apps/logger/migrations/0033_auto_20170705_0159.py index 3cc2db0cb1..e2f30634da 100644 --- a/onadata/apps/logger/migrations/0033_auto_20170705_0159.py +++ b/onadata/apps/logger/migrations/0033_auto_20170705_0159.py @@ -8,28 +8,30 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0032_project_deleted_at'), + ("logger", "0032_project_deleted_at"), ] operations = [ migrations.AlterModelOptions( - name='attachment', - options={'ordering': ('pk',)}, + name="attachment", + options={"ordering": ("pk",)}, ), migrations.AlterModelOptions( - name='xform', + name="xform", options={ - 'ordering': ('pk',), - 'permissions': - (('view_xform', 'Can view associated data'), - ('view_xform_all', 'Can view all associated data'), - ('view_xform_data', 'Can view submitted data'), - ('report_xform', 'Can make submissions to the form'), - ('move_xform', 'Can move form between projects'), - ('transfer_xform', 'Can transfer form ownership.'), - ('can_export_xform_data', 'Can export form data'), - ('delete_submission', 'Can delete submissions from form')), - 'verbose_name': 'XForm', - 'verbose_name_plural': 'XForms'}, + "ordering": ("pk",), + "permissions": ( + ("view_xform", "Can view associated data"), + ("view_xform_all", "Can view all associated data"), + ("view_xform_data", "Can view submitted data"), + ("report_xform", "Can make submissions to the form"), + ("move_xform", "Can move form between projects"), + ("transfer_xform", "Can transfer form ownership."), + ("can_export_xform_data", "Can export form data"), + ("delete_submission", "Can delete submissions from form"), + ), + "verbose_name": "XForm", + "verbose_name_plural": "XForms", + }, ), ] diff --git a/onadata/apps/logger/migrations/0034_auto_20170814_0432.py b/onadata/apps/logger/migrations/0034_auto_20170814_0432.py index 89841453ed..4d4f32ec01 100644 --- a/onadata/apps/logger/migrations/0034_auto_20170814_0432.py +++ b/onadata/apps/logger/migrations/0034_auto_20170814_0432.py @@ -8,32 +8,29 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0033_auto_20170705_0159'), + ("logger", "0033_auto_20170705_0159"), ] operations = [ migrations.AddField( - model_name='instance', - name='media_all_received', + model_name="instance", + name="media_all_received", field=models.NullBooleanField( - null=True, - verbose_name='Received All Media Attachemts' + null=True, verbose_name="Received All Media Attachemts" ), ), migrations.AddField( - model_name='instance', - name='media_count', + model_name="instance", + name="media_count", field=models.PositiveIntegerField( - null=True, - verbose_name='Received Media Attachments' + null=True, verbose_name="Received Media Attachments" ), ), migrations.AddField( - model_name='instance', - name='total_media', + model_name="instance", + name="total_media", field=models.PositiveIntegerField( - null=True, - verbose_name='Total Media Attachments' + null=True, verbose_name="Total Media Attachments" ), ), ] diff --git a/onadata/apps/logger/migrations/0034_mergedxform.py b/onadata/apps/logger/migrations/0034_mergedxform.py index 7d68dc8566..fdea7177d1 100644 --- a/onadata/apps/logger/migrations/0034_mergedxform.py +++ b/onadata/apps/logger/migrations/0034_mergedxform.py @@ -8,21 +8,32 @@ class Migration(migrations.Migration): - dependencies = [('logger', '0033_auto_20170705_0159'), ] + dependencies = [ + ("logger", "0033_auto_20170705_0159"), + ] operations = [ migrations.CreateModel( - name='MergedXForm', + name="MergedXForm", fields=[ - ('xform_ptr', models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to='logger.XForm')), - ('xforms', models.ManyToManyField( - related_name='mergedxform_ptr', to='logger.XForm')), + ( + "xform_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="logger.XForm", + ), + ), + ( + "xforms", + models.ManyToManyField( + related_name="mergedxform_ptr", to="logger.XForm" + ), + ), ], - bases=('logger.xform', ), ), + bases=("logger.xform",), + ), ] diff --git a/onadata/apps/logger/migrations/0035_auto_20170712_0529.py b/onadata/apps/logger/migrations/0035_auto_20170712_0529.py index 6a4578e620..5861f6861e 100644 --- a/onadata/apps/logger/migrations/0035_auto_20170712_0529.py +++ b/onadata/apps/logger/migrations/0035_auto_20170712_0529.py @@ -7,13 +7,15 @@ class Migration(migrations.Migration): - dependencies = [('logger', '0034_mergedxform'), ] + dependencies = [ + ("logger", "0034_mergedxform"), + ] operations = [ migrations.AlterModelOptions( - name='mergedxform', + name="mergedxform", options={ - 'permissions': (('view_mergedxform', 'Can view associated data' - ), ) - }, ), + "permissions": (("view_mergedxform", "Can view associated data"),) + }, + ), ] diff --git a/onadata/apps/logger/migrations/0036_xform_is_merged_dataset.py b/onadata/apps/logger/migrations/0036_xform_is_merged_dataset.py index 4705454762..2e80fa3c62 100644 --- a/onadata/apps/logger/migrations/0036_xform_is_merged_dataset.py +++ b/onadata/apps/logger/migrations/0036_xform_is_merged_dataset.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0035_auto_20170712_0529'), + ("logger", "0035_auto_20170712_0529"), ] operations = [ migrations.AddField( - model_name='xform', - name='is_merged_dataset', + model_name="xform", + name="is_merged_dataset", field=models.BooleanField(default=False), ), ] diff --git a/onadata/apps/logger/migrations/0037_merge_20170825_0238.py b/onadata/apps/logger/migrations/0037_merge_20170825_0238.py index e6d9231e4c..8b385e0258 100644 --- a/onadata/apps/logger/migrations/0037_merge_20170825_0238.py +++ b/onadata/apps/logger/migrations/0037_merge_20170825_0238.py @@ -8,9 +8,8 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0034_auto_20170814_0432'), - ('logger', '0036_xform_is_merged_dataset'), + ("logger", "0034_auto_20170814_0432"), + ("logger", "0036_xform_is_merged_dataset"), ] - operations = [ - ] + operations = [] diff --git a/onadata/apps/logger/migrations/0038_auto_20170828_1718.py b/onadata/apps/logger/migrations/0038_auto_20170828_1718.py index 3ecb6b1668..028faf59cf 100644 --- a/onadata/apps/logger/migrations/0038_auto_20170828_1718.py +++ b/onadata/apps/logger/migrations/0038_auto_20170828_1718.py @@ -8,26 +8,29 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0037_merge_20170825_0238'), + ("logger", "0037_merge_20170825_0238"), ] operations = [ migrations.AlterField( - model_name='instance', - name='media_all_received', + model_name="instance", + name="media_all_received", field=models.NullBooleanField( - default=True, verbose_name='Received All Media Attachemts'), ), + default=True, verbose_name="Received All Media Attachemts" + ), + ), migrations.AlterField( - model_name='instance', - name='media_count', + model_name="instance", + name="media_count", field=models.PositiveIntegerField( - default=0, - null=True, - verbose_name='Received Media Attachments'), ), + default=0, null=True, verbose_name="Received Media Attachments" + ), + ), migrations.AlterField( - model_name='instance', - name='total_media', + model_name="instance", + name="total_media", field=models.PositiveIntegerField( - default=0, null=True, verbose_name='Total Media Attachments'), + default=0, null=True, verbose_name="Total Media Attachments" + ), ), ] diff --git a/onadata/apps/logger/migrations/0039_auto_20170909_2052.py b/onadata/apps/logger/migrations/0039_auto_20170909_2052.py index 8ba7bffabb..bc2197a7ae 100644 --- a/onadata/apps/logger/migrations/0039_auto_20170909_2052.py +++ b/onadata/apps/logger/migrations/0039_auto_20170909_2052.py @@ -8,18 +8,18 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0038_auto_20170828_1718'), + ("logger", "0038_auto_20170828_1718"), ] operations = [ migrations.AddField( - model_name='instance', - name='checksum', + model_name="instance", + name="checksum", field=models.CharField(blank=True, max_length=64, null=True), ), migrations.AddField( - model_name='instancehistory', - name='checksum', + model_name="instancehistory", + name="checksum", field=models.CharField(blank=True, max_length=64, null=True), ), ] diff --git a/onadata/apps/logger/migrations/0040_auto_20170912_1504.py b/onadata/apps/logger/migrations/0040_auto_20170912_1504.py index 22c6225315..2c2d0b96e0 100644 --- a/onadata/apps/logger/migrations/0040_auto_20170912_1504.py +++ b/onadata/apps/logger/migrations/0040_auto_20170912_1504.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0039_auto_20170909_2052'), + ("logger", "0039_auto_20170909_2052"), ] operations = [ migrations.AlterField( - model_name='instance', - name='checksum', - field=models.CharField( - blank=True, db_index=True, max_length=64, null=True), ), + model_name="instance", + name="checksum", + field=models.CharField(blank=True, db_index=True, max_length=64, null=True), + ), ] diff --git a/onadata/apps/logger/migrations/0041_auto_20170912_1512.py b/onadata/apps/logger/migrations/0041_auto_20170912_1512.py index fb082d79d0..a107435ce4 100644 --- a/onadata/apps/logger/migrations/0041_auto_20170912_1512.py +++ b/onadata/apps/logger/migrations/0041_auto_20170912_1512.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0040_auto_20170912_1504'), + ("logger", "0040_auto_20170912_1504"), ] operations = [ migrations.AlterField( - model_name='instance', - name='uuid', - field=models.CharField(db_index=True, default='', max_length=249), + model_name="instance", + name="uuid", + field=models.CharField(db_index=True, default="", max_length=249), ), ] diff --git a/onadata/apps/logger/migrations/0042_xform_hash.py b/onadata/apps/logger/migrations/0042_xform_hash.py index 8814600606..c6628b7b52 100644 --- a/onadata/apps/logger/migrations/0042_xform_hash.py +++ b/onadata/apps/logger/migrations/0042_xform_hash.py @@ -8,14 +8,15 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0041_auto_20170912_1512'), + ("logger", "0041_auto_20170912_1512"), ] operations = [ migrations.AddField( - model_name='xform', - name='hash', - field=models.CharField(blank=True, default=None, max_length=32, - null=True, verbose_name='Hash'), + model_name="xform", + name="hash", + field=models.CharField( + blank=True, default=None, max_length=32, null=True, verbose_name="Hash" + ), ), ] diff --git a/onadata/apps/logger/migrations/0043_auto_20171010_0403.py b/onadata/apps/logger/migrations/0043_auto_20171010_0403.py index a1da0b61ad..41e7118d4b 100644 --- a/onadata/apps/logger/migrations/0043_auto_20171010_0403.py +++ b/onadata/apps/logger/migrations/0043_auto_20171010_0403.py @@ -8,14 +8,15 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0042_xform_hash'), + ("logger", "0042_xform_hash"), ] operations = [ migrations.AlterField( - model_name='xform', - name='hash', - field=models.CharField(blank=True, default=None, max_length=36, - null=True, verbose_name='Hash'), + model_name="xform", + name="hash", + field=models.CharField( + blank=True, default=None, max_length=36, null=True, verbose_name="Hash" + ), ), ] diff --git a/onadata/apps/logger/migrations/0044_xform_hash_sql_update.py b/onadata/apps/logger/migrations/0044_xform_hash_sql_update.py index 079550fd67..9823b90623 100644 --- a/onadata/apps/logger/migrations/0044_xform_hash_sql_update.py +++ b/onadata/apps/logger/migrations/0044_xform_hash_sql_update.py @@ -7,12 +7,12 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0043_auto_20171010_0403'), + ("logger", "0043_auto_20171010_0403"), ] operations = [ migrations.RunSQL( "UPDATE logger_xform SET hash = CONCAT('md5:', MD5(XML)) WHERE hash IS NULL;", # noqa - migrations.RunSQL.noop + migrations.RunSQL.noop, ), ] diff --git a/onadata/apps/logger/migrations/0045_attachment_name.py b/onadata/apps/logger/migrations/0045_attachment_name.py index 33fcc5858a..a46dd78552 100644 --- a/onadata/apps/logger/migrations/0045_attachment_name.py +++ b/onadata/apps/logger/migrations/0045_attachment_name.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0044_xform_hash_sql_update'), + ("logger", "0044_xform_hash_sql_update"), ] operations = [ migrations.AddField( - model_name='attachment', - name='name', + model_name="attachment", + name="name", field=models.CharField(blank=True, max_length=100, null=True), ), ] diff --git a/onadata/apps/logger/migrations/0046_auto_20180314_1618.py b/onadata/apps/logger/migrations/0046_auto_20180314_1618.py index 982674e63a..f342b68690 100644 --- a/onadata/apps/logger/migrations/0046_auto_20180314_1618.py +++ b/onadata/apps/logger/migrations/0046_auto_20180314_1618.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0045_attachment_name'), + ("logger", "0045_attachment_name"), ] operations = [ migrations.AlterField( - model_name='xform', - name='uuid', - field=models.CharField(default='', max_length=36), + model_name="xform", + name="uuid", + field=models.CharField(default="", max_length=36), ), ] diff --git a/onadata/apps/logger/migrations/0047_dataview_deleted_at.py b/onadata/apps/logger/migrations/0047_dataview_deleted_at.py index 5342aa5bf8..1072becbb1 100644 --- a/onadata/apps/logger/migrations/0047_dataview_deleted_at.py +++ b/onadata/apps/logger/migrations/0047_dataview_deleted_at.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0046_auto_20180314_1618'), + ("logger", "0046_auto_20180314_1618"), ] operations = [ migrations.AddField( - model_name='dataview', - name='deleted_at', + model_name="dataview", + name="deleted_at", field=models.DateTimeField(blank=True, null=True), ), ] diff --git a/onadata/apps/logger/migrations/0048_dataview_deleted_by.py b/onadata/apps/logger/migrations/0048_dataview_deleted_by.py index 5822b374f8..bacacb0154 100644 --- a/onadata/apps/logger/migrations/0048_dataview_deleted_by.py +++ b/onadata/apps/logger/migrations/0048_dataview_deleted_by.py @@ -11,19 +11,20 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0047_dataview_deleted_at'), + ("logger", "0047_dataview_deleted_at"), ] operations = [ migrations.AddField( - model_name='dataview', - name='deleted_by', + model_name="dataview", + name="deleted_by", field=models.ForeignKey( blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, - related_name='dataview_deleted_by', - to=settings.AUTH_USER_MODEL), + related_name="dataview_deleted_by", + to=settings.AUTH_USER_MODEL, + ), ), ] diff --git a/onadata/apps/logger/migrations/0049_xform_deleted_by.py b/onadata/apps/logger/migrations/0049_xform_deleted_by.py index 3c862e2118..a79ec9de0d 100644 --- a/onadata/apps/logger/migrations/0049_xform_deleted_by.py +++ b/onadata/apps/logger/migrations/0049_xform_deleted_by.py @@ -11,18 +11,20 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0048_dataview_deleted_by'), + ("logger", "0048_dataview_deleted_by"), ] operations = [ migrations.AddField( - model_name='xform', - name='deleted_by', + model_name="xform", + name="deleted_by", field=models.ForeignKey( blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, - related_name='xform_deleted_by', to=settings.AUTH_USER_MODEL), + related_name="xform_deleted_by", + to=settings.AUTH_USER_MODEL, + ), ), ] diff --git a/onadata/apps/logger/migrations/0050_project_deleted_by.py b/onadata/apps/logger/migrations/0050_project_deleted_by.py index e96a4d29dc..5fd8889b11 100644 --- a/onadata/apps/logger/migrations/0050_project_deleted_by.py +++ b/onadata/apps/logger/migrations/0050_project_deleted_by.py @@ -11,17 +11,20 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0049_xform_deleted_by'), + ("logger", "0049_xform_deleted_by"), ] operations = [ migrations.AddField( - model_name='project', - name='deleted_by', + model_name="project", + name="deleted_by", field=models.ForeignKey( - blank=True, default=None, null=True, + blank=True, + default=None, + null=True, on_delete=django.db.models.deletion.SET_NULL, - related_name='project_deleted_by', - to=settings.AUTH_USER_MODEL), + related_name="project_deleted_by", + to=settings.AUTH_USER_MODEL, + ), ), ] diff --git a/onadata/apps/logger/migrations/0051_auto_20180522_1118.py b/onadata/apps/logger/migrations/0051_auto_20180522_1118.py index 24b4520e26..7e8067b2bd 100644 --- a/onadata/apps/logger/migrations/0051_auto_20180522_1118.py +++ b/onadata/apps/logger/migrations/0051_auto_20180522_1118.py @@ -15,15 +15,16 @@ def recalculate_xform_hash(apps, schema_editor): # pylint: disable=W0613 """ Recalculate all XForm hashes. """ - XForm = apps.get_model('logger', 'XForm') # pylint: disable=C0103 - xforms = XForm.objects.filter(downloadable=True, - deleted_at__isnull=True).only('xml') + XForm = apps.get_model("logger", "XForm") # pylint: disable=C0103 + xforms = XForm.objects.filter(downloadable=True, deleted_at__isnull=True).only( + "xml" + ) count = xforms.count() counter = 0 for xform in queryset_iterator(xforms, 500): - xform.hash = u'md5:%s' % md5(xform.xml.encode('utf8')).hexdigest() - xform.save(update_fields=['hash']) + xform.hash = "md5:%s" % md5(xform.xml.encode("utf8")).hexdigest() + xform.save(update_fields=["hash"]) counter += 1 if counter % 500 == 0: print("Processed %d of %d forms." % (counter, count)) @@ -37,9 +38,7 @@ class Migration(migrations.Migration): """ dependencies = [ - ('logger', '0050_project_deleted_by'), + ("logger", "0050_project_deleted_by"), ] - operations = [ - migrations.RunPython(recalculate_xform_hash) - ] + operations = [migrations.RunPython(recalculate_xform_hash)] diff --git a/onadata/apps/logger/migrations/0052_auto_20180805_2233.py b/onadata/apps/logger/migrations/0052_auto_20180805_2233.py index 364a048101..147a589ae5 100644 --- a/onadata/apps/logger/migrations/0052_auto_20180805_2233.py +++ b/onadata/apps/logger/migrations/0052_auto_20180805_2233.py @@ -14,17 +14,18 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0051_auto_20180522_1118'), + ("logger", "0051_auto_20180522_1118"), ] operations = [ migrations.AddField( - model_name='instance', - name='deleted_by', + model_name="instance", + name="deleted_by", field=models.ForeignKey( null=True, on_delete=django.db.models.deletion.CASCADE, - related_name='deleted_instances', - to=settings.AUTH_USER_MODEL), + related_name="deleted_instances", + to=settings.AUTH_USER_MODEL, + ), ), ] diff --git a/onadata/apps/logger/migrations/0053_submissionreview.py b/onadata/apps/logger/migrations/0053_submissionreview.py index 555127c75b..000e768ce5 100644 --- a/onadata/apps/logger/migrations/0053_submissionreview.py +++ b/onadata/apps/logger/migrations/0053_submissionreview.py @@ -11,56 +11,77 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0052_auto_20180805_2233'), + ("logger", "0052_auto_20180805_2233"), ] operations = [ migrations.CreateModel( - name='SubmissionReview', + name="SubmissionReview", fields=[ - ('id', - models.AutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name='ID')), - ('status', - models.CharField( - choices=[('1', 'Approved'), ('3', 'Pending'), - ('2', 'Rejected')], - db_index=True, - default='3', - max_length=1, - verbose_name='Status')), - ('deleted_at', models.DateTimeField(default=None, null=True)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('created_by', - models.ForeignKey( - blank=True, - default=None, - null=True, - on_delete=django.db.models.deletion.CASCADE, - to=settings.AUTH_USER_MODEL)), - ('deleted_by', - models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name='deleted_reviews', - to=settings.AUTH_USER_MODEL)), - ('instance', - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name='reviews', - to='logger.Instance')), - ('note', - models.ForeignKey( - blank=True, - default=None, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name='notes', - to='logger.Note')), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "status", + models.CharField( + choices=[ + ("1", "Approved"), + ("3", "Pending"), + ("2", "Rejected"), + ], + db_index=True, + default="3", + max_length=1, + verbose_name="Status", + ), + ), + ("deleted_at", models.DateTimeField(default=None, null=True)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "created_by", + models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "deleted_by", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="deleted_reviews", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "instance", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="reviews", + to="logger.Instance", + ), + ), + ( + "note", + models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="notes", + to="logger.Note", + ), + ), ], ), ] diff --git a/onadata/apps/logger/migrations/0054_instance_has_a_review.py b/onadata/apps/logger/migrations/0054_instance_has_a_review.py index 739b52dff0..27ca162a9f 100644 --- a/onadata/apps/logger/migrations/0054_instance_has_a_review.py +++ b/onadata/apps/logger/migrations/0054_instance_has_a_review.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0053_submissionreview'), + ("logger", "0053_submissionreview"), ] operations = [ migrations.AddField( - model_name='instance', - name='has_a_review', - field=models.BooleanField(default=False, verbose_name='has_a_review'), + model_name="instance", + name="has_a_review", + field=models.BooleanField(default=False, verbose_name="has_a_review"), ), ] diff --git a/onadata/apps/logger/migrations/0055_auto_20180904_0713.py b/onadata/apps/logger/migrations/0055_auto_20180904_0713.py index 19d80a1dbd..ee8710f90d 100644 --- a/onadata/apps/logger/migrations/0055_auto_20180904_0713.py +++ b/onadata/apps/logger/migrations/0055_auto_20180904_0713.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0054_instance_has_a_review'), + ("logger", "0054_instance_has_a_review"), ] operations = [ migrations.AlterField( - model_name='submissionreview', - name='deleted_at', + model_name="submissionreview", + name="deleted_at", field=models.DateTimeField(db_index=True, default=None, null=True), ), ] diff --git a/onadata/apps/logger/migrations/0056_auto_20190125_0517.py b/onadata/apps/logger/migrations/0056_auto_20190125_0517.py index 2b2b51e7f9..a485c5c9ce 100644 --- a/onadata/apps/logger/migrations/0056_auto_20190125_0517.py +++ b/onadata/apps/logger/migrations/0056_auto_20190125_0517.py @@ -9,79 +9,130 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0055_auto_20180904_0713'), + ("logger", "0055_auto_20180904_0713"), ] operations = [ migrations.AlterModelOptions( - name='mergedxform', + name="mergedxform", options={}, ), migrations.AlterModelOptions( - name='note', + name="note", options={}, ), migrations.AlterModelOptions( - name='project', - options={'permissions': (('add_project_xform', 'Can add xform to project'), ('report_project_xform', 'Can make submissions to the project'), ('transfer_project', 'Can transfer project to different owner'), ('can_export_project_data', 'Can export data in project'), ('view_project_all', 'Can view all associated data'), ('view_project_data', 'Can view submitted data'))}, + name="project", + options={ + "permissions": ( + ("add_project_xform", "Can add xform to project"), + ("report_project_xform", "Can make submissions to the project"), + ("transfer_project", "Can transfer project to different owner"), + ("can_export_project_data", "Can export data in project"), + ("view_project_all", "Can view all associated data"), + ("view_project_data", "Can view submitted data"), + ) + }, ), migrations.AlterModelOptions( - name='xform', - options={'ordering': ('pk',), 'permissions': (('view_xform_all', 'Can view all associated data'), ('view_xform_data', 'Can view submitted data'), ('report_xform', 'Can make submissions to the form'), ('move_xform', 'Can move form between projects'), ('transfer_xform', 'Can transfer form ownership.'), ('can_export_xform_data', 'Can export form data'), ('delete_submission', 'Can delete submissions from form')), 'verbose_name': 'XForm', 'verbose_name_plural': 'XForms'}, + name="xform", + options={ + "ordering": ("pk",), + "permissions": ( + ("view_xform_all", "Can view all associated data"), + ("view_xform_data", "Can view submitted data"), + ("report_xform", "Can make submissions to the form"), + ("move_xform", "Can move form between projects"), + ("transfer_xform", "Can transfer form ownership."), + ("can_export_xform_data", "Can export form data"), + ("delete_submission", "Can delete submissions from form"), + ), + "verbose_name": "XForm", + "verbose_name_plural": "XForms", + }, ), migrations.AlterField( - model_name='attachment', - name='mimetype', - field=models.CharField(blank=True, default='', max_length=100), + model_name="attachment", + name="mimetype", + field=models.CharField(blank=True, default="", max_length=100), ), migrations.AlterField( - model_name='instance', - name='deleted_by', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='deleted_instances', to=settings.AUTH_USER_MODEL), + model_name="instance", + name="deleted_by", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="deleted_instances", + to=settings.AUTH_USER_MODEL, + ), ), migrations.AlterField( - model_name='instance', - name='survey_type', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='logger.SurveyType'), + model_name="instance", + name="survey_type", + field=models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, to="logger.SurveyType" + ), ), migrations.AlterField( - model_name='instance', - name='user', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='instances', to=settings.AUTH_USER_MODEL), + model_name="instance", + name="user", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="instances", + to=settings.AUTH_USER_MODEL, + ), ), migrations.AlterField( - model_name='osmdata', - name='field_name', - field=models.CharField(blank=True, default='', max_length=255), + model_name="osmdata", + name="field_name", + field=models.CharField(blank=True, default="", max_length=255), ), migrations.AlterField( - model_name='osmdata', - name='osm_type', - field=models.CharField(default='way', max_length=10), + model_name="osmdata", + name="osm_type", + field=models.CharField(default="way", max_length=10), ), migrations.AlterField( - model_name='project', - name='tags', - field=taggit.managers.TaggableManager(help_text='A comma-separated list of tags.', related_name='project_tags', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags'), + model_name="project", + name="tags", + field=taggit.managers.TaggableManager( + help_text="A comma-separated list of tags.", + related_name="project_tags", + through="taggit.TaggedItem", + to="taggit.Tag", + verbose_name="Tags", + ), ), migrations.AlterField( - model_name='widget', - name='order', - field=models.PositiveIntegerField(db_index=True, editable=False, verbose_name='order'), + model_name="widget", + name="order", + field=models.PositiveIntegerField( + db_index=True, editable=False, verbose_name="order" + ), ), migrations.AlterField( - model_name='widget', - name='widget_type', - field=models.CharField(choices=[('charts', 'Charts')], default='charts', max_length=25), + model_name="widget", + name="widget_type", + field=models.CharField( + choices=[("charts", "Charts")], default="charts", max_length=25 + ), ), migrations.AlterField( - model_name='xform', - name='created_by', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL), + model_name="xform", + name="created_by", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + ), ), migrations.AlterField( - model_name='xform', - name='sms_id_string', - field=models.SlugField(default='', editable=False, max_length=100, verbose_name='SMS ID'), + model_name="xform", + name="sms_id_string", + field=models.SlugField( + default="", editable=False, max_length=100, verbose_name="SMS ID" + ), ), ] diff --git a/onadata/apps/logger/migrations/0057_xform_public_key.py b/onadata/apps/logger/migrations/0057_xform_public_key.py index 8d1e12848b..8c4054561e 100644 --- a/onadata/apps/logger/migrations/0057_xform_public_key.py +++ b/onadata/apps/logger/migrations/0057_xform_public_key.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0056_auto_20190125_0517'), + ("logger", "0056_auto_20190125_0517"), ] operations = [ migrations.AddField( - model_name='xform', - name='public_key', - field=models.TextField(default=''), + model_name="xform", + name="public_key", + field=models.TextField(default=""), ), ] diff --git a/onadata/apps/logger/migrations/0058_auto_20191211_0900.py b/onadata/apps/logger/migrations/0058_auto_20191211_0900.py index 778a00a2e7..5be3a64f78 100644 --- a/onadata/apps/logger/migrations/0058_auto_20191211_0900.py +++ b/onadata/apps/logger/migrations/0058_auto_20191211_0900.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0057_xform_public_key'), + ("logger", "0057_xform_public_key"), ] operations = [ migrations.AlterField( - model_name='xform', - name='public_key', - field=models.TextField(blank=True, default='', null=True), + model_name="xform", + name="public_key", + field=models.TextField(blank=True, default="", null=True), ), ] diff --git a/onadata/apps/logger/migrations/0059_attachment_deleted_by.py b/onadata/apps/logger/migrations/0059_attachment_deleted_by.py index 4a0cd2ac86..fbe84ad48c 100644 --- a/onadata/apps/logger/migrations/0059_attachment_deleted_by.py +++ b/onadata/apps/logger/migrations/0059_attachment_deleted_by.py @@ -9,13 +9,18 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0058_auto_20191211_0900'), + ("logger", "0058_auto_20191211_0900"), ] operations = [ migrations.AddField( - model_name='attachment', - name='deleted_by', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='deleted_attachments', to=settings.AUTH_USER_MODEL), + model_name="attachment", + name="deleted_by", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="deleted_attachments", + to=settings.AUTH_USER_MODEL, + ), ), ] diff --git a/onadata/apps/logger/migrations/0060_auto_20200305_0357.py b/onadata/apps/logger/migrations/0060_auto_20200305_0357.py index 4efeb9f57a..182082c2a1 100644 --- a/onadata/apps/logger/migrations/0060_auto_20200305_0357.py +++ b/onadata/apps/logger/migrations/0060_auto_20200305_0357.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0059_attachment_deleted_by'), + ("logger", "0059_attachment_deleted_by"), ] operations = [ migrations.AlterField( - model_name='xform', - name='uuid', - field=models.CharField(db_index=True, default='', max_length=36), + model_name="xform", + name="uuid", + field=models.CharField(db_index=True, default="", max_length=36), ), ] diff --git a/onadata/apps/logger/migrations/0061_auto_20200713_0814.py b/onadata/apps/logger/migrations/0061_auto_20200713_0814.py index 239cc02c2b..f67ecc8941 100644 --- a/onadata/apps/logger/migrations/0061_auto_20200713_0814.py +++ b/onadata/apps/logger/migrations/0061_auto_20200713_0814.py @@ -9,9 +9,9 @@ def generate_uuid_if_missing(apps, schema_editor): """ Generate uuids for XForms without them """ - XForm = apps.get_model('logger', 'XForm') + XForm = apps.get_model("logger", "XForm") - for xform in XForm.objects.filter(uuid=''): + for xform in XForm.objects.filter(uuid=""): xform.uuid = get_uuid() xform.save() @@ -19,8 +19,7 @@ def generate_uuid_if_missing(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('logger', '0060_auto_20200305_0357'), + ("logger", "0060_auto_20200305_0357"), ] - operations = [ - migrations.RunPython(generate_uuid_if_missing)] + operations = [migrations.RunPython(generate_uuid_if_missing)] diff --git a/onadata/apps/logger/migrations/0062_auto_20210202_0248.py b/onadata/apps/logger/migrations/0062_auto_20210202_0248.py index f545fdd69a..0dcfa1989d 100644 --- a/onadata/apps/logger/migrations/0062_auto_20210202_0248.py +++ b/onadata/apps/logger/migrations/0062_auto_20210202_0248.py @@ -10,9 +10,10 @@ def regenerate_instance_json(apps, schema_editor): Regenerate Instance JSON """ for inst in Instance.objects.filter( - deleted_at__isnull=True, - xform__downloadable=True, - xform__deleted_at__isnull=True): + deleted_at__isnull=True, + xform__downloadable=True, + xform__deleted_at__isnull=True, + ): inst.json = inst.get_full_dict(load_existing=False) inst.save() @@ -20,9 +21,7 @@ def regenerate_instance_json(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('logger', '0061_auto_20200713_0814'), + ("logger", "0061_auto_20200713_0814"), ] - operations = [ - migrations.RunPython(regenerate_instance_json) - ] + operations = [migrations.RunPython(regenerate_instance_json)] diff --git a/onadata/apps/logger/migrations/0063_xformversion.py b/onadata/apps/logger/migrations/0063_xformversion.py index 064469a5d3..d19c0543b2 100644 --- a/onadata/apps/logger/migrations/0063_xformversion.py +++ b/onadata/apps/logger/migrations/0063_xformversion.py @@ -9,25 +9,47 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0062_auto_20210202_0248'), + ("logger", "0062_auto_20210202_0248"), ] operations = [ migrations.CreateModel( - name='XFormVersion', + name="XFormVersion", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('xls', models.FileField(upload_to='')), - ('version', models.CharField(max_length=100)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('xml', models.TextField()), - ('json', models.TextField()), - ('created_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), - ('xform', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='logger.XForm')), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("xls", models.FileField(upload_to="")), + ("version", models.CharField(max_length=100)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ("xml", models.TextField()), + ("json", models.TextField()), + ( + "created_by", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "xform", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="versions", + to="logger.XForm", + ), + ), ], options={ - 'unique_together': {('xform', 'version')}, + "unique_together": {("xform", "version")}, }, ), ] diff --git a/onadata/apps/logger/migrations/0064_auto_20210304_0314.py b/onadata/apps/logger/migrations/0064_auto_20210304_0314.py index 94b712f87e..59a0653dc4 100644 --- a/onadata/apps/logger/migrations/0064_auto_20210304_0314.py +++ b/onadata/apps/logger/migrations/0064_auto_20210304_0314.py @@ -11,10 +11,7 @@ def create_initial_xform_version(apps, schema_editor): Creates an XFormVersion object for an XForm that has no Version """ - queryset = XForm.objects.filter( - downloadable=True, - deleted_at__isnull=True - ) + queryset = XForm.objects.filter(downloadable=True, deleted_at__isnull=True) for xform in queryset.iterator(): if xform.version: create_xform_version(xform, xform.user) @@ -23,9 +20,7 @@ def create_initial_xform_version(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('logger', '0063_xformversion'), + ("logger", "0063_xformversion"), ] - operations = [ - migrations.RunPython(create_initial_xform_version) - ] + operations = [migrations.RunPython(create_initial_xform_version)] diff --git a/onadata/apps/logger/models/__init__.py b/onadata/apps/logger/models/__init__.py index 6dab34889d..7d85c6d6be 100644 --- a/onadata/apps/logger/models/__init__.py +++ b/onadata/apps/logger/models/__init__.py @@ -1,14 +1,18 @@ +# -*- coding: utf-8 -*- +""" +Logger models. +""" from onadata.apps.logger.models.attachment import Attachment # noqa from onadata.apps.logger.models.data_view import DataView # noqa from onadata.apps.logger.models.instance import Instance # noqa from onadata.apps.logger.models.merged_xform import MergedXForm # noqa -from onadata.apps.logger.models.note import Note # noqa -from onadata.apps.logger.models.open_data import OpenData # noqa -from onadata.apps.logger.models.osmdata import OsmData # noqa -from onadata.apps.logger.models.project import Project # noqa -from onadata.apps.logger.models.survey_type import SurveyType # noqa -from onadata.apps.logger.models.widget import Widget # noqa -from onadata.apps.logger.models.xform import XForm # noqa -from onadata.apps.logger.models.submission_review import SubmissionReview # noqa -from onadata.apps.logger.xform_instance_parser import InstanceParseError # noqa -from onadata.apps.logger.models.xform_version import XFormVersion # noqa +from onadata.apps.logger.models.note import Note # noqa +from onadata.apps.logger.models.open_data import OpenData # noqa +from onadata.apps.logger.models.osmdata import OsmData # noqa +from onadata.apps.logger.models.project import Project # noqa +from onadata.apps.logger.models.survey_type import SurveyType # noqa +from onadata.apps.logger.models.widget import Widget # noqa +from onadata.apps.logger.models.xform import XForm # noqa +from onadata.apps.logger.models.submission_review import SubmissionReview # noqa +from onadata.apps.logger.xform_instance_parser import InstanceParseError # noqa +from onadata.apps.logger.models.xform_version import XFormVersion # noqa diff --git a/onadata/apps/logger/models/attachment.py b/onadata/apps/logger/models/attachment.py index 9ea5520bfb..5a58a186b3 100644 --- a/onadata/apps/logger/models/attachment.py +++ b/onadata/apps/logger/models/attachment.py @@ -1,12 +1,18 @@ +# -*- coding: utf-8 -*- +""" +Attachment model. +""" +import hashlib import mimetypes import os -from hashlib import md5 +from django.contrib.auth import get_user_model from django.db import models -from django.contrib.auth.models import User def get_original_filename(filename): + """Returns the filename removing the hashed random string added to it when we have + file duplicates in some file systems.""" # https://docs.djangoproject.com/en/1.8/ref/files/storage/ # #django.core.files.storage.Storage.get_available_name # If a file with name already exists, an underscore plus a random @@ -14,23 +20,27 @@ def get_original_filename(filename): # before the extension. # this code trys to reverse this effect to derive the original name if filename: - parts = filename.split('_') + parts = filename.split("_") if len(parts) > 1: - ext_parts = parts[-1].split('.') + ext_parts = parts[-1].split(".") if len(ext_parts[0]) == 7 and len(ext_parts) == 2: ext = ext_parts[1] - return u'.'.join([u'_'.join(parts[:-1]), ext]) + return ".".join(["_".join(parts[:-1]), ext]) return filename def upload_to(instance, filename): - folder = "{}_{}".format(instance.instance.xform.id, - instance.instance.xform.id_string) + """Returns the attachments folder to upload the file to.""" + folder = f"{instance.instance.xform.id}_{instance.instance.xform.id_string}" + return os.path.join( - instance.instance.xform.user.username, 'attachments', folder, - os.path.split(filename)[1]) + instance.instance.xform.user.username, + "attachments", + folder, + os.path.split(filename)[1], + ) class Attachment(models.Model): @@ -38,37 +48,40 @@ class Attachment(models.Model): Attachment model. """ - OSM = 'osm' + OSM = "osm" instance = models.ForeignKey( - 'logger.Instance', related_name="attachments", - on_delete=models.CASCADE) + "logger.Instance", related_name="attachments", on_delete=models.CASCADE + ) media_file = models.FileField(max_length=255, upload_to=upload_to) - mimetype = models.CharField( - max_length=100, null=False, blank=True, default='') - extension = models.CharField(max_length=10, null=False, blank=False, - default=u"non", db_index=True) + mimetype = models.CharField(max_length=100, null=False, blank=True, default="") + extension = models.CharField( + max_length=10, null=False, blank=False, default="non", db_index=True + ) date_created = models.DateTimeField(null=True, auto_now_add=True) date_modified = models.DateTimeField(null=True, auto_now=True) deleted_at = models.DateTimeField(null=True, default=None) file_size = models.PositiveIntegerField(default=0) name = models.CharField(max_length=100, null=True, blank=True) - deleted_by = models.ForeignKey(User, related_name='deleted_attachments', - null=True, on_delete=models.SET_NULL) + deleted_by = models.ForeignKey( + get_user_model(), + related_name="deleted_attachments", + null=True, + on_delete=models.SET_NULL, + ) class Meta: - app_label = 'logger' - ordering = ("pk", ) + app_label = "logger" + ordering = ("pk",) def save(self, *args, **kwargs): - if self.media_file and self.mimetype == '': + if self.media_file and self.mimetype == "": # guess mimetype - mimetype, encoding = mimetypes.guess_type(self.media_file.name) + mimetype, _encoding = mimetypes.guess_type(self.media_file.name) if mimetype: self.mimetype = mimetype if self.media_file and len(self.media_file.name) > 255: - raise ValueError( - "Length of the media file should be less or equal to 255") + raise ValueError("Length of the media file should be less or equal to 255") try: f_size = self.media_file.size @@ -77,15 +90,22 @@ def save(self, *args, **kwargs): except (OSError, AttributeError): pass - super(Attachment, self).save(*args, **kwargs) + super().save(*args, **kwargs) @property def file_hash(self): + """Returns the MD5 hash of the file.""" + md5_hash = "" if self.media_file.storage.exists(self.media_file.name): - return u'%s' % md5(self.media_file.read()).hexdigest() - return u'' + md5_hash = hashlib.new( + "md5", self.media_file.read(), usedforsecurity=False + ).hexdigest() + return md5_hash @property def filename(self): + """Returns the attachment's filename.""" + filename = "" if self.media_file: - return os.path.basename(self.media_file.name) + filename = os.path.basename(self.media_file.name) + return filename diff --git a/onadata/apps/logger/models/data_view.py b/onadata/apps/logger/models/data_view.py index 0721fb46dd..21a8525c68 100644 --- a/onadata/apps/logger/models/data_view.py +++ b/onadata/apps/logger/models/data_view.py @@ -3,45 +3,54 @@ DataView model class """ import datetime -from builtins import str as text +import json from django.conf import settings from django.contrib.gis.db import models -from django.contrib.postgres.fields import JSONField from django.db import connection from django.db.models.signals import post_delete, post_save from django.utils import timezone -from django.utils.encoding import python_2_unicode_compatible -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ +from django.db.utils import DataError from onadata.apps.viewer.parsed_instance_tools import get_where_clause -from onadata.libs.models.sorting import (json_order_by, json_order_by_params, - sort_from_mongo_sort_str) -from onadata.libs.utils.cache_tools import (DATAVIEW_COUNT, - DATAVIEW_LAST_SUBMISSION_TIME, - XFORM_LINKED_DATAVIEWS, - PROJ_OWNER_CACHE, - safe_delete) -from onadata.libs.utils.common_tags import (ATTACHMENTS, EDITED, GEOLOCATION, - ID, LAST_EDITED, MONGO_STRFTIME, - NOTES, SUBMISSION_TIME) - -SUPPORTED_FILTERS = ['=', '>', '<', '>=', '<=', '<>', '!='] -ATTACHMENT_TYPES = ['photo', 'audio', 'video'] -DEFAULT_COLUMNS = [ - ID, SUBMISSION_TIME, EDITED, LAST_EDITED, NOTES] - - -def _json_sql_str(key, known_integers=None, known_dates=None, - known_decimals=None): - _json_str = u"json->>%s" +from onadata.libs.models.sorting import ( # noqa pylint: disable=unused-import + json_order_by, + json_order_by_params, + sort_from_mongo_sort_str, +) +from onadata.libs.utils.cache_tools import ( # noqa pylint: disable=unused-import + DATAVIEW_COUNT, + DATAVIEW_LAST_SUBMISSION_TIME, + XFORM_LINKED_DATAVIEWS, + PROJ_OWNER_CACHE, + safe_delete, +) +from onadata.libs.utils.common_tags import ( + ATTACHMENTS, + EDITED, + GEOLOCATION, + ID, + LAST_EDITED, + MONGO_STRFTIME, + NOTES, + SUBMISSION_TIME, +) + +SUPPORTED_FILTERS = ["=", ">", "<", ">=", "<=", "<>", "!="] +ATTACHMENT_TYPES = ["photo", "audio", "video"] +DEFAULT_COLUMNS = [ID, SUBMISSION_TIME, EDITED, LAST_EDITED, NOTES] + + +def _json_sql_str(key, known_integers=None, known_dates=None, known_decimals=None): + _json_str = "json->>%s" if known_integers is not None and key in known_integers: - _json_str = u"CAST(json->>%s AS INT)" + _json_str = "CAST(json->>%s AS INT)" elif known_dates is not None and key in known_dates: - _json_str = u"CAST(json->>%s AS TIMESTAMP)" + _json_str = "CAST(json->>%s AS TIMESTAMP)" elif known_decimals is not None and key in known_decimals: - _json_str = u"CAST(JSON->>%s AS DECIMAL)" + _json_str = "CAST(JSON->>%s AS DECIMAL)" return _json_str @@ -51,10 +60,10 @@ def get_name_from_survey_element(element): def append_where_list(comp, t_list, json_str): - if comp in ['=', '>', '<', '>=', '<=']: - t_list.append(u"{} {} %s".format(json_str, comp)) - elif comp in ['<>', '!=']: - t_list.append(u"{} <> %s".format(json_str)) + if comp in ["=", ">", "<", ">=", "<="]: + t_list.append(f"{json_str} {comp}" + " %s") + elif comp in ["<>", "!="]: + t_list.append(f"{json_str} <>" + " %s") return t_list @@ -63,8 +72,7 @@ def get_elements_of_type(xform, field_type): """ This function returns a list of column names of a specified type """ - return [f.get('name') - for f in xform.get_survey_elements_of_type(field_type)] + return [f.get("name") for f in xform.get_survey_elements_of_type(field_type)] def has_attachments_fields(data_view): @@ -81,39 +89,42 @@ def has_attachments_fields(data_view): attachments += get_elements_of_type(xform, element_type) if attachments: - for a in data_view.columns: - if a in attachments: + for attachment in data_view.columns: + if attachment in attachments: return True return False -@python_2_unicode_compatible class DataView(models.Model): """ Model to provide filtered access to the underlying data of an XForm """ name = models.CharField(max_length=255) - xform = models.ForeignKey('logger.XForm', on_delete=models.CASCADE) - project = models.ForeignKey('logger.Project', on_delete=models.CASCADE) + xform = models.ForeignKey("logger.XForm", on_delete=models.CASCADE) + project = models.ForeignKey("logger.Project", on_delete=models.CASCADE) - columns = JSONField() - query = JSONField(default=dict, blank=True) + columns = models.JSONField() + query = models.JSONField(default=dict, blank=True) instances_with_geopoints = models.BooleanField(default=False) matches_parent = models.BooleanField(default=False) date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) deleted_at = models.DateTimeField(blank=True, null=True) - deleted_by = models.ForeignKey(settings.AUTH_USER_MODEL, - related_name='dataview_deleted_by', - null=True, on_delete=models.SET_NULL, - default=None, blank=True) + deleted_by = models.ForeignKey( + settings.AUTH_USER_MODEL, + related_name="dataview_deleted_by", + null=True, + on_delete=models.SET_NULL, + default=None, + blank=True, + ) class Meta: - app_label = 'logger' - verbose_name = _('Data View') - verbose_name_plural = _('Data Views') + app_label = "logger" + verbose_name = _("Data View") + verbose_name_plural = _("Data Views") def __str__(self): return getattr(self, "name", "") @@ -126,7 +137,7 @@ def has_geo_columnn_n_data(self): # Get the form geo xpaths xform = self.xform - geo_xpaths = xform.geopoint_xpaths() # pylint: disable=E1101 + geo_xpaths = xform.geopoint_xpaths() set_geom = set(geo_xpaths) set_columns = set(self.columns) @@ -140,50 +151,52 @@ def has_geo_columnn_n_data(self): def save(self, *args, **kwargs): self.instances_with_geopoints = self.has_geo_columnn_n_data() - return super(DataView, self).save(*args, **kwargs) + return super().save(*args, **kwargs) def _get_known_type(self, type_str): - # pylint: disable=E1101 return [ get_name_from_survey_element(e) - for e in self.xform.get_survey_elements_of_type(type_str)] + for e in self.xform.get_survey_elements_of_type(type_str) + ] def get_known_integers(self): """Return elements of type integer""" - return self._get_known_type('integer') + return self._get_known_type("integer") def get_known_dates(self): """Return elements of type date""" - return self._get_known_type('date') + return self._get_known_type("date") def get_known_decimals(self): """Return elements of type decimal""" - return self._get_known_type('decimal') + return self._get_known_type("decimal") def has_instance(self, instance): """Return True if instance in set of dataview data""" cursor = connection.cursor() - sql = u"SELECT count(json) FROM logger_instance" - - where, where_params = self._get_where_clause(self, - self.get_known_integers(), - self.get_known_dates(), - self.get_known_decimals()) - sql_where = u"" + sql = "SELECT count(json) FROM logger_instance" + + where, where_params = self._get_where_clause( + self, + self.get_known_integers(), + self.get_known_dates(), + self.get_known_decimals(), + ) + sql_where = "" if where: - sql_where = u" AND " + u" AND ".join(where) + sql_where = " AND " + " AND ".join(where) - sql += u" WHERE xform_id = %s AND id = %s" + sql_where \ - + u" AND deleted_at IS NULL" - # pylint: disable=E1101 + sql += ( + " WHERE xform_id = %s AND id = %s" + sql_where + " AND deleted_at IS NULL" + ) params = [self.xform.pk, instance.id] + where_params - cursor.execute(sql, [text(i) for i in params]) - + cursor.execute(sql, [str(i) for i in params]) + records = None for row in cursor.fetchall(): records = row[0] - return True if records else False + return records is not None def soft_delete(self, user=None): """ @@ -192,51 +205,59 @@ def soft_delete(self, user=None): uniqueness constraint. """ soft_deletion_time = timezone.now() - deletion_suffix = soft_deletion_time.strftime('-deleted-at-%s') + deletion_suffix = soft_deletion_time.strftime("-deleted-at-%s") self.deleted_at = soft_deletion_time self.name += deletion_suffix - update_fields = ['date_modified', 'deleted_at', 'name', 'deleted_by'] + update_fields = ["date_modified", "deleted_at", "name", "deleted_by"] if user is not None: self.deleted_by = user - update_fields.append('deleted_by') + update_fields.append("deleted_by") self.save(update_fields=update_fields) @classmethod - def _get_where_clause(cls, data_view, form_integer_fields=[], - form_date_fields=[], form_decimal_fields=[]): - known_integers = ['_id'] + form_integer_fields - known_dates = ['_submission_time'] + form_date_fields + def _get_where_clause( # pylint: disable=too-many-locals + cls, + data_view, + form_integer_fields=None, + form_date_fields=None, + form_decimal_fields=None, + ): + form_integer_fields = [] if form_integer_fields is None else form_integer_fields + form_date_fields = [] if form_date_fields is None else form_date_fields + form_decimal_fields = [] if form_decimal_fields is None else form_decimal_fields + known_integers = ["_id"] + form_integer_fields + known_dates = ["_submission_time"] + form_date_fields known_decimals = form_decimal_fields where = [] where_params = [] - query = data_view.query + queries = data_view.query or_where = [] or_params = [] - for qu in query: - comp = qu.get('filter') - column = qu.get('column') - value = qu.get('value') - condi = qu.get('condition') + for query in queries: + comp = query.get("filter") + column = query.get("column") + value = query.get("value") + condi = query.get("condition") - json_str = _json_sql_str(column, known_integers, known_dates, - known_decimals) + json_str = _json_sql_str( + column, known_integers, known_dates, known_decimals + ) if comp in known_dates: - value = datetime.datetime.strptime( - value[:19], MONGO_STRFTIME) + value = datetime.datetime.strptime(value[:19], MONGO_STRFTIME) - if condi and condi.lower() == 'or': + if condi and condi.lower() == "or": or_where = append_where_list(comp, or_where, json_str) - or_params.extend((column, text(value))) + or_params.extend((column, str(value))) else: where = append_where_list(comp, where, json_str) - where_params.extend((column, text(value))) + where_params.extend((column, str(value))) if or_where: - or_where = [u"".join([u"(", u" OR ".join(or_where), u")"])] + or_where = ["".join(["(", " OR ".join(or_where), ")"])] where += or_where where_params.extend(or_params) @@ -244,46 +265,60 @@ def _get_where_clause(cls, data_view, form_integer_fields=[], return where, where_params @classmethod - def query_iterator(cls, sql, fields=None, params=[], count=False): + def query_iterator(cls, sql, fields=None, params=None, count=False): + def parse_json(data): + try: + return json.loads(data) + except ValueError: + return data + + params = [] if params is None else params + cursor = connection.cursor() - sql_params = tuple( - i if isinstance(i, tuple) else text(i) for i in params) + sql_params = tuple(i if isinstance(i, tuple) else str(i) for i in params) if count: - from_pos = sql.upper().find(' FROM') + from_pos = sql.upper().find(" FROM") if from_pos != -1: - sql = u"SELECT COUNT(*) " + sql[from_pos:] + sql = "SELECT COUNT(*) " + sql[from_pos:] - order_pos = sql.upper().find('ORDER BY') + order_pos = sql.upper().find("ORDER BY") if order_pos != -1: sql = sql[:order_pos] - fields = [u'count'] + fields = ["count"] cursor.execute(sql, sql_params) if fields is None: for row in cursor.fetchall(): - yield row[0] + yield parse_json(row[0]) else: if count: for row in cursor.fetchall(): yield dict(zip(fields, row)) else: for row in cursor.fetchall(): - yield dict(zip(fields, [row[0].get(f) for f in fields])) + yield dict(zip(fields, [parse_json(row[0]).get(f) for f in fields])) + # pylint: disable=too-many-arguments,too-many-locals,too-many-branches @classmethod - def generate_query_string(cls, data_view, start_index, limit, - last_submission_time, all_data, sort, - filter_query=None): - additional_columns = [GEOLOCATION] \ - if data_view.instances_with_geopoints else [] + def generate_query_string( + cls, + data_view, + start_index, + limit, + last_submission_time, + all_data, + sort, + filter_query=None, + ): + additional_columns = [GEOLOCATION] if data_view.instances_with_geopoints else [] if has_attachments_fields(data_view): additional_columns += [ATTACHMENTS] - sql = u"SELECT json FROM logger_instance" + sql = "SELECT json FROM logger_instance" if all_data or data_view.matches_parent: columns = None elif last_submission_time: @@ -300,12 +335,15 @@ def generate_query_string(cls, data_view, start_index, limit, data_view, data_view.get_known_integers(), data_view.get_known_dates(), - data_view.get_known_decimals()) + data_view.get_known_decimals(), + ) if filter_query: - add_where, add_where_params = \ - get_where_clause(filter_query, data_view.get_known_integers(), - data_view.get_known_decimals()) + add_where, add_where_params = get_where_clause( + filter_query, + data_view.get_known_integers(), + data_view.get_known_decimals(), + ) if add_where: where = where + add_where @@ -313,79 +351,91 @@ def generate_query_string(cls, data_view, start_index, limit, sql_where = "" if where: - sql_where = u" AND " + u" AND ".join(where) + sql_where = " AND " + " AND ".join(where) if data_view.xform.is_merged_dataset: - sql += u" WHERE xform_id IN %s " + sql_where \ - + u" AND deleted_at IS NULL" - params = [tuple(list( - data_view.xform.mergedxform.xforms.values_list('pk', flat=True) - ))] + where_params + sql += " WHERE xform_id IN %s " + sql_where + " AND deleted_at IS NULL" + params = [ + tuple( + list( + data_view.xform.mergedxform.xforms.values_list("pk", flat=True) + ) + ) + ] + where_params else: - sql += u" WHERE xform_id = %s " + sql_where \ - + u" AND deleted_at IS NULL" + sql += " WHERE xform_id = %s " + sql_where + " AND deleted_at IS NULL" params = [data_view.xform.pk] + where_params if sort is not None: - sort = ['id'] if sort is None\ - else sort_from_mongo_sort_str(sort) - sql = u"{} {}".format(sql, json_order_by(sort)) + sort = ["id"] if sort is None else sort_from_mongo_sort_str(sort) + sql = f"{sql} {json_order_by(sort)}" params = params + json_order_by_params(sort) elif last_submission_time is False: - sql += ' ORDER BY id' + sql += " ORDER BY id" if start_index is not None: - sql += u" OFFSET %s" + sql += " OFFSET %s" params += [start_index] if limit is not None: - sql += u" LIMIT %s" + sql += " LIMIT %s" params += [limit] if last_submission_time: - sql += u" ORDER BY date_created DESC" - sql += u" LIMIT 1" + sql += " ORDER BY date_created DESC" + sql += " LIMIT 1" - return (sql, columns, params, ) + return ( + sql, + columns, + params, + ) @classmethod - def query_data(cls, data_view, start_index=None, limit=None, count=None, - last_submission_time=False, all_data=False, sort=None, - filter_query=None): + def query_data( # pylint: disable=too-many-arguments + cls, + data_view, + start_index=None, + limit=None, + count=None, + last_submission_time=False, + all_data=False, + sort=None, + filter_query=None, + ): + """Returns a list of records for the view based on the parameters passed in.""" (sql, columns, params) = cls.generate_query_string( - data_view, start_index, limit, last_submission_time, - all_data, sort, filter_query) + data_view, + start_index, + limit, + last_submission_time, + all_data, + sort, + filter_query, + ) try: - records = [record for record in DataView.query_iterator(sql, - columns, - params, - count)] - except Exception as e: - return {"error": _(text(e))} + records = list(DataView.query_iterator(sql, columns, params, count)) + except DataError as e: + return {"error": _(str(e))} return records -def clear_cache(sender, instance, **kwargs): - """ Post delete handler for clearing the dataview cache. - """ - safe_delete('{}{}'.format(XFORM_LINKED_DATAVIEWS, instance.xform.pk)) +def clear_cache(sender, instance, **kwargs): # pylint: disable=unused-argument + """Post delete handler for clearing the dataview cache.""" + safe_delete(f"{XFORM_LINKED_DATAVIEWS}{instance.xform.pk}") -def clear_dataview_cache(sender, instance, **kwargs): - """ Post Save handler for clearing dataview cache on serialized fields. - """ - safe_delete('{}{}'.format(PROJ_OWNER_CACHE, instance.project.pk)) - safe_delete('{}{}'.format(DATAVIEW_COUNT, instance.xform.pk)) - safe_delete( - '{}{}'.format(DATAVIEW_LAST_SUBMISSION_TIME, instance.xform.pk)) - safe_delete('{}{}'.format(XFORM_LINKED_DATAVIEWS, instance.xform.pk)) +def clear_dataview_cache(sender, instance, **kwargs): # pylint: disable=unused-argument + """Post Save handler for clearing dataview cache on serialized fields.""" + safe_delete(f"{PROJ_OWNER_CACHE}{instance.project.pk}") + safe_delete(f"{DATAVIEW_COUNT}{instance.xform.pk}") + safe_delete(f"{DATAVIEW_LAST_SUBMISSION_TIME}{instance.xform.pk}") + safe_delete(f"{XFORM_LINKED_DATAVIEWS}{instance.xform.pk}") -post_save.connect(clear_dataview_cache, sender=DataView, - dispatch_uid='clear_cache') +post_save.connect(clear_dataview_cache, sender=DataView, dispatch_uid="clear_cache") -post_delete.connect(clear_cache, sender=DataView, - dispatch_uid='clear_xform_cache') +post_delete.connect(clear_cache, sender=DataView, dispatch_uid="clear_xform_cache") diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index ff85e527ce..dc29cb6566 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -3,78 +3,113 @@ Instance model class """ import math -import pytz from datetime import datetime -from deprecated import deprecated from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.contrib.gis.db import models from django.contrib.gis.geos import GeometryCollection, Point -from django.contrib.postgres.fields import JSONField from django.core.cache import cache from django.db import connection, transaction from django.db.models import Q from django.db.models.signals import post_delete, post_save from django.urls import reverse from django.utils import timezone -from django.utils.translation import ugettext as _ -from future.utils import python_2_unicode_compatible -from past.builtins import basestring # pylint: disable=W0622 +from django.utils.translation import gettext as _ + +import pytz +from deprecated import deprecated from taggit.managers import TaggableManager from onadata.apps.logger.models.submission_review import SubmissionReview from onadata.apps.logger.models.survey_type import SurveyType from onadata.apps.logger.models.xform import XFORM_TITLE_LENGTH, XForm -from onadata.apps.logger.xform_instance_parser import (XFormInstanceParser, - clean_and_parse_xml, - get_uuid_from_xml) +from onadata.apps.logger.xform_instance_parser import ( + XFormInstanceParser, + clean_and_parse_xml, + get_uuid_from_xml, +) from onadata.celery import app from onadata.libs.data.query import get_numeric_fields from onadata.libs.utils.cache_tools import ( - DATAVIEW_COUNT, IS_ORG, PROJ_NUM_DATASET_CACHE, PROJ_SUB_DATE_CACHE, - XFORM_COUNT, XFORM_DATA_VERSIONS, XFORM_SUBMISSION_COUNT_FOR_DAY, - XFORM_SUBMISSION_COUNT_FOR_DAY_DATE, safe_delete) + DATAVIEW_COUNT, + IS_ORG, + PROJ_NUM_DATASET_CACHE, + PROJ_SUB_DATE_CACHE, + XFORM_COUNT, + XFORM_DATA_VERSIONS, + XFORM_SUBMISSION_COUNT_FOR_DAY, + XFORM_SUBMISSION_COUNT_FOR_DAY_DATE, + safe_delete, +) from onadata.libs.utils.common_tags import ( - ATTACHMENTS, BAMBOO_DATASET_ID, DATE_MODIFIED, - DELETEDAT, DURATION, EDITED, END, GEOLOCATION, ID, LAST_EDITED, - MEDIA_ALL_RECEIVED, MEDIA_COUNT, MONGO_STRFTIME, NOTES, - REVIEW_STATUS, START, STATUS, SUBMISSION_TIME, SUBMITTED_BY, - TAGS, TOTAL_MEDIA, UUID, VERSION, XFORM_ID, XFORM_ID_STRING, - REVIEW_COMMENT, REVIEW_DATE) + ATTACHMENTS, + BAMBOO_DATASET_ID, + DATE_MODIFIED, + DELETEDAT, + DURATION, + EDITED, + END, + GEOLOCATION, + ID, + LAST_EDITED, + MEDIA_ALL_RECEIVED, + MEDIA_COUNT, + MONGO_STRFTIME, + NOTES, + REVIEW_COMMENT, + REVIEW_DATE, + REVIEW_STATUS, + START, + STATUS, + SUBMISSION_TIME, + SUBMITTED_BY, + TAGS, + TOTAL_MEDIA, + UUID, + VERSION, + XFORM_ID, + XFORM_ID_STRING, +) from onadata.libs.utils.dict_tools import get_values_matching_key from onadata.libs.utils.model_tools import set_uuid from onadata.libs.utils.timing import calculate_duration -ASYNC_POST_SUBMISSION_PROCESSING_ENABLED = \ - getattr(settings, 'ASYNC_POST_SUBMISSION_PROCESSING_ENABLED', False) +ASYNC_POST_SUBMISSION_PROCESSING_ENABLED = getattr( + settings, "ASYNC_POST_SUBMISSION_PROCESSING_ENABLED", False +) +# pylint: disable=invalid-name +User = get_user_model() def get_attachment_url(attachment, suffix=None): - kwargs = {'pk': attachment.pk} - url = u'{}?filename={}'.format( - reverse('files-detail', kwargs=kwargs), - attachment.media_file.name + """ + Returns the attachment URL for a given suffix + """ + kwargs = {"pk": attachment.pk} + url = ( + f"{reverse('files-detail', kwargs=kwargs)}" + f"?filename={attachment.media_file.name}" ) if suffix: - url += u'&suffix={}'.format(suffix) + url += f"&suffix={suffix}" return url def _get_attachments_from_instance(instance): attachments = [] - for a in instance.attachments.filter(deleted_at__isnull=True): - attachment = dict() - attachment['download_url'] = get_attachment_url(a) - attachment['small_download_url'] = get_attachment_url(a, 'small') - attachment['medium_download_url'] = get_attachment_url(a, 'medium') - attachment['mimetype'] = a.mimetype - attachment['filename'] = a.media_file.name - attachment['name'] = a.name - attachment['instance'] = a.instance.pk - attachment['xform'] = instance.xform.id - attachment['id'] = a.id + for item in instance.attachments.filter(deleted_at__isnull=True): + attachment = {} + attachment["download_url"] = get_attachment_url(item) + attachment["small_download_url"] = get_attachment_url(item, "small") + attachment["medium_download_url"] = get_attachment_url(item, "medium") + attachment["mimetype"] = item.mimetype + attachment["filename"] = item.media_file.name + attachment["name"] = item.name + attachment["instance"] = item.instance.pk + attachment["xform"] = instance.xform.id + attachment["id"] = item.id attachments.append(attachment) return attachments @@ -86,21 +121,24 @@ def _get_tag_or_element_type_xpath(xform, tag): return elems[0].get_abbreviated_xpath() if elems else tag -@python_2_unicode_compatible class FormInactiveError(Exception): """Exception class for inactive forms""" + def __str__(self): - return _(u'Form is inactive') + return _("Form is inactive") -@python_2_unicode_compatible class FormIsMergedDatasetError(Exception): """Exception class for merged datasets""" + def __str__(self): - return _(u'Submissions are not allowed on merged datasets.') + return _("Submissions are not allowed on merged datasets.") def numeric_checker(string_value): + """ + Checks if a ``string_value`` is a numeric value. + """ try: return int(string_value) except ValueError: @@ -113,46 +151,49 @@ def numeric_checker(string_value): return string_value + # need to establish id_string of the xform before we run get_dict since # we now rely on data dictionary to parse the xml def get_id_string_from_xml_str(xml_str): + """ + Parses an XML ``xml_str`` and returns the top level id string. + """ xml_obj = clean_and_parse_xml(xml_str) root_node = xml_obj.documentElement - id_string = root_node.getAttribute(u"id") + id_string = root_node.getAttribute("id") - if len(id_string) == 0: + if not id_string: # may be hidden in submission/data/id_string - elems = root_node.getElementsByTagName('data') + elems = root_node.getElementsByTagName("data") for data in elems: - for child in data.childNodes: - id_string = data.childNodes[0].getAttribute('id') + id_string = data.childNodes[0].getAttribute("id") - if len(id_string) > 0: - break - - if len(id_string) > 0: + if id_string: break return id_string def submission_time(): + """Returns current timestamp via timezone.now().""" return timezone.now() def _update_submission_count_for_today( - form_id: int, incr: bool = True, date_created=None): + form_id: int, incr: bool = True, date_created=None +): # Track submissions made today current_timzone_name = timezone.get_current_timezone_name() current_timezone = pytz.timezone(current_timzone_name) today = datetime.today() current_date = current_timezone.localize( - datetime(today.year, today.month, today.day)).isoformat() - date_cache_key = (f"{XFORM_SUBMISSION_COUNT_FOR_DAY_DATE}" f"{form_id}") - count_cache_key = (f"{XFORM_SUBMISSION_COUNT_FOR_DAY}{form_id}") + datetime(today.year, today.month, today.day) + ).isoformat() + date_cache_key = f"{XFORM_SUBMISSION_COUNT_FOR_DAY_DATE}" f"{form_id}" + count_cache_key = f"{XFORM_SUBMISSION_COUNT_FOR_DAY}{form_id}" if not cache.get(date_cache_key) == current_date: cache.set(date_cache_key, current_date, 86400) @@ -174,80 +215,87 @@ def _update_submission_count_for_today( @app.task @transaction.atomic() def update_xform_submission_count(instance_id, created): + """Updates the XForm submissions count on a new submission being created.""" if created: + + # pylint: disable=import-outside-toplevel from multidb.pinning import use_master + with use_master: try: - instance = Instance.objects.select_related('xform').only( - 'xform__user_id', 'date_created').get(pk=instance_id) + instance = ( + Instance.objects.select_related("xform") + .only("xform__user_id", "date_created") + .get(pk=instance_id) + ) except Instance.DoesNotExist: pass else: # update xform.num_of_submissions cursor = connection.cursor() sql = ( - 'UPDATE logger_xform SET ' - 'num_of_submissions = num_of_submissions + 1, ' - 'last_submission_time = %s ' - 'WHERE id = %s' + "UPDATE logger_xform SET " + "num_of_submissions = num_of_submissions + 1, " + "last_submission_time = %s " + "WHERE id = %s" ) params = [instance.date_created, instance.xform_id] # update user profile.num_of_submissions cursor.execute(sql, params) sql = ( - 'UPDATE main_userprofile SET ' - 'num_of_submissions = num_of_submissions + 1 ' - 'WHERE user_id = %s' + "UPDATE main_userprofile SET " + "num_of_submissions = num_of_submissions + 1 " + "WHERE user_id = %s" ) cursor.execute(sql, [instance.xform.user_id]) # Track submissions made today _update_submission_count_for_today(instance.xform_id) - safe_delete('{}{}'.format( - XFORM_DATA_VERSIONS, instance.xform_id)) - safe_delete('{}{}'.format(DATAVIEW_COUNT, instance.xform_id)) - safe_delete('{}{}'.format(XFORM_COUNT, instance.xform_id)) + safe_delete(f"{XFORM_DATA_VERSIONS}{instance.xform_id}") + safe_delete(f"{DATAVIEW_COUNT}{instance.xform_id}") + safe_delete(f"{XFORM_COUNT}{instance.xform_id}") # Clear project cache - from onadata.apps.logger.models.xform import \ - clear_project_cache + from onadata.apps.logger.models.xform import clear_project_cache + clear_project_cache(instance.xform.project_id) +# pylint: disable=unused-argument,invalid-name def update_xform_submission_count_delete(sender, instance, **kwargs): + """Updates the XForm submissions count on deletion of a submission.""" try: xform = XForm.objects.select_for_update().get(pk=instance.xform.pk) except XForm.DoesNotExist: pass else: xform.num_of_submissions -= 1 - if xform.num_of_submissions < 0: - xform.num_of_submissions = 0 - xform.save(update_fields=['num_of_submissions']) + + xform.num_of_submissions = max(xform.num_of_submissions, 0) + xform.save(update_fields=["num_of_submissions"]) profile_qs = User.profile.get_queryset() try: - profile = profile_qs.select_for_update()\ - .get(pk=xform.user.profile.pk) + profile = profile_qs.select_for_update().get(pk=xform.user.profile.pk) except profile_qs.model.DoesNotExist: pass else: profile.num_of_submissions -= 1 - if profile.num_of_submissions < 0: - profile.num_of_submissions = 0 + profile.num_of_submissions = max(profile.num_of_submissions, 0) profile.save() # Track submissions made today _update_submission_count_for_today( - xform.id, incr=False, date_created=instance.date_created) + xform.id, incr=False, date_created=instance.date_created + ) - for a in [PROJ_NUM_DATASET_CACHE, PROJ_SUB_DATE_CACHE]: - safe_delete('{}{}'.format(a, xform.project.pk)) + for cache_prefix in [PROJ_NUM_DATASET_CACHE, PROJ_SUB_DATE_CACHE]: + safe_delete(f"{cache_prefix}{xform.project.pk}") - safe_delete('{}{}'.format(IS_ORG, xform.pk)) - safe_delete('{}{}'.format(XFORM_DATA_VERSIONS, xform.pk)) - safe_delete('{}{}'.format(DATAVIEW_COUNT, xform.pk)) - safe_delete('{}{}'.format(XFORM_COUNT, xform.pk)) + safe_delete(f"{IS_ORG}{xform.pk}") + safe_delete(f"{XFORM_DATA_VERSIONS}{xform.pk}") + safe_delete(f"{DATAVIEW_COUNT}{xform.pk}") + safe_delete(f"{XFORM_COUNT}{xform.pk}") if xform.instances.exclude(geom=None).count() < 1: xform.instances_with_geopoints = False @@ -264,60 +312,70 @@ def save_full_json(instance_id, created): pass else: instance.json = instance.get_full_dict() - instance.save(update_fields=['json']) + instance.save(update_fields=["json"]) +# pylint: disable=unused-argument @app.task def update_project_date_modified(instance_id, created): + """Update the project's date_modified + + Changes the etag value of the projects endpoint. + """ # update the date modified field of the project which will change # the etag value of the projects endpoint try: - instance = Instance.objects.select_related('xform__project').only( - 'xform__project__date_modified').get(pk=instance_id) + instance = ( + Instance.objects.select_related("xform__project") + .only("xform__project__date_modified") + .get(pk=instance_id) + ) except Instance.DoesNotExist: pass else: - instance.xform.project.save(update_fields=['date_modified']) + instance.xform.project.save(update_fields=["date_modified"]) def convert_to_serializable_date(date): - if hasattr(date, 'isoformat'): + """Returns the ISO format of a date object if it has the attribute 'isoformat'.""" + if hasattr(date, "isoformat"): return date.isoformat() return date -class InstanceBaseClass(object): +class InstanceBaseClass: """Interface of functions for Instance and InstanceHistory model""" @property def point(self): - gc = self.geom + """Returns the Point of the first geom if it is a collection.""" + geom_collection = self.geom - if gc and len(gc): - return gc[0] + if geom_collection and geom_collection.num_points: + return geom_collection[0] + return self.geom def numeric_converter(self, json_dict, numeric_fields=None): + """Converts strings in a python object ``json_dict`` to their numeric value.""" if numeric_fields is None: # pylint: disable=no-member numeric_fields = get_numeric_fields(self.xform) for key, value in json_dict.items(): - if isinstance(value, basestring) and key in numeric_fields: + if isinstance(value, str) and key in numeric_fields: converted_value = numeric_checker(value) if converted_value: json_dict[key] = converted_value elif isinstance(value, dict): - json_dict[key] = self.numeric_converter( - value, numeric_fields) + json_dict[key] = self.numeric_converter(value, numeric_fields) elif isinstance(value, list): for k, v in enumerate(value): - if isinstance(v, basestring) and key in numeric_fields: + if isinstance(v, str) and key in numeric_fields: converted_value = numeric_checker(v) if converted_value: json_dict[key] = converted_value elif isinstance(v, dict): - value[k] = self.numeric_converter( - v, numeric_fields) + value[k] = self.numeric_converter(v, numeric_fields) return json_dict def _set_geom(self): @@ -337,37 +395,42 @@ def _set_geom(self): except ValueError: return - if not xform.instances_with_geopoints and len(points): + if not xform.instances_with_geopoints and points: xform.instances_with_geopoints = True xform.save() + # pylint: disable=attribute-defined-outside-init self.geom = GeometryCollection(points) def _set_json(self): + # pylint: disable=attribute-defined-outside-init self.json = self.get_full_dict() def get_full_dict(self, load_existing=True): + """Returns the submission XML as a python dictionary object.""" doc = self.json or {} if load_existing else {} # Get latest dict doc = self.get_dict() # pylint: disable=no-member if self.id: - doc.update({ - UUID: self.uuid, - ID: self.id, - BAMBOO_DATASET_ID: self.xform.bamboo_dataset, - ATTACHMENTS: _get_attachments_from_instance(self), - STATUS: self.status, - TAGS: list(self.tags.names()), - NOTES: self.get_notes(), - VERSION: self.version, - DURATION: self.get_duration(), - XFORM_ID_STRING: self._parser.get_xform_id_string(), - XFORM_ID: self.xform.pk, - GEOLOCATION: [self.point.y, self.point.x] if self.point - else [None, None], - SUBMITTED_BY: self.user.username if self.user else None - }) + geopoint = [self.point.y, self.point.x] if self.point else [None, None] + doc.update( + { + UUID: self.uuid, + ID: self.id, + BAMBOO_DATASET_ID: self.xform.bamboo_dataset, + ATTACHMENTS: _get_attachments_from_instance(self), + STATUS: self.status, + TAGS: list(self.tags.names()), + NOTES: self.get_notes(), + VERSION: self.version, + DURATION: self.get_duration(), + XFORM_ID_STRING: self._parser.get_xform_id_string(), + XFORM_ID: self.xform.pk, + GEOLOCATION: geopoint, + SUBMITTED_BY: self.user.username if self.user else None, + } + ) for osm in self.osm_data.all(): doc.update(osm.get_tags_with_prefix()) @@ -380,20 +443,19 @@ def get_full_dict(self, load_existing=True): review = self.get_latest_review() if review: doc[REVIEW_STATUS] = review.status - doc[REVIEW_DATE] = review.date_created.strftime( - MONGO_STRFTIME) + doc[REVIEW_DATE] = review.date_created.strftime(MONGO_STRFTIME) if review.get_note_text(): doc[REVIEW_COMMENT] = review.get_note_text() - # pylint: disable=attribute-defined-outside-init + # pylint: disable=attribute-defined-outside-init,access-member-before-definition if not self.date_created: self.date_created = submission_time() + # pylint: disable=access-member-before-definition if not self.date_modified: self.date_modified = self.date_created - doc[DATE_MODIFIED] = self.date_modified.strftime( - MONGO_STRFTIME) + doc[DATE_MODIFIED] = self.date_modified.strftime(MONGO_STRFTIME) doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME) @@ -402,26 +464,30 @@ def get_full_dict(self, load_existing=True): doc[MEDIA_ALL_RECEIVED] = self.media_all_received edited = False - if hasattr(self, 'last_edited'): + if hasattr(self, "last_edited"): edited = self.last_edited is not None doc[EDITED] = edited - edited and doc.update({ - LAST_EDITED: convert_to_serializable_date(self.last_edited) - }) + if edited: + doc.update( + {LAST_EDITED: convert_to_serializable_date(self.last_edited)} + ) return doc def _set_parser(self): if not hasattr(self, "_parser"): # pylint: disable=no-member + # pylint: disable=attribute-defined-outside-init self._parser = XFormInstanceParser(self.xml, self.xform) def _set_survey_type(self): - self.survey_type, created = \ - SurveyType.objects.get_or_create(slug=self.get_root_node_name()) + # pylint: disable=attribute-defined-outside-init + self.survey_type, _created = SurveyType.objects.get_or_create( + slug=self.get_root_node_name() + ) def _set_uuid(self): - # pylint: disable=no-member, attribute-defined-outside-init + # pylint: disable=no-member,attribute-defined-outside-init,access-member-before-definition if self.xml and not self.uuid: # pylint: disable=no-member uuid = get_uuid_from_xml(self.xml) @@ -430,44 +496,52 @@ def _set_uuid(self): set_uuid(self) def get(self, abbreviated_xpath): + """Returns the XML element at the ``abbreviated_xpath``.""" self._set_parser() return self._parser.get(abbreviated_xpath) + # pylint: disable=unused-argument def get_dict(self, force_new=False, flat=True): """Return a python object representation of this instance's XML.""" self._set_parser() - instance_dict = self._parser.get_flat_dict_with_attributes() if flat \ + instance_dict = ( + self._parser.get_flat_dict_with_attributes() + if flat else self._parser.to_dict() + ) return self.numeric_converter(instance_dict) def get_notes(self): + """Returns a list of notes.""" # pylint: disable=no-member return [note.get_data() for note in self.notes.all()] - @deprecated(version='2.5.3', - reason="Deprecated in favour of `get_latest_review`") + @deprecated(version="2.5.3", reason="Deprecated in favour of `get_latest_review`") def get_review_status_and_comment(self): """ Return a tuple of review status and comment. """ try: # pylint: disable=no-member - status = self.reviews.latest('date_modified').status - comment = self.reviews.latest('date_modified').get_note_text() + status = self.reviews.latest("date_modified").status + comment = self.reviews.latest("date_modified").get_note_text() return status, comment except SubmissionReview.DoesNotExist: return None def get_root_node(self): + """Returns the XML submission's root node.""" self._set_parser() return self._parser.get_root_node() def get_root_node_name(self): + """Returns the XML submission's root node name.""" self._set_parser() return self._parser.get_root_node_name() def get_duration(self): + """Returns the duration between the `start` and `end` questions of a form.""" data = self.get_dict() # pylint: disable=no-member start_name = _get_tag_or_element_type_xpath(self.xform, START) @@ -482,7 +556,8 @@ def get_latest_review(self): Used in favour of `get_review_status_and_comment`. """ try: - return self.reviews.latest('date_modified') + # pylint: disable=no-member + return self.reviews.latest("date_modified") except SubmissionReview.DoesNotExist: return None @@ -492,15 +567,15 @@ class Instance(models.Model, InstanceBaseClass): Model representing a single submission to an XForm """ - json = JSONField(default=dict, null=False) + json = models.JSONField(default=dict, null=False) xml = models.TextField() user = models.ForeignKey( - User, related_name='instances', null=True, on_delete=models.SET_NULL) + User, related_name="instances", null=True, on_delete=models.SET_NULL + ) xform = models.ForeignKey( - 'logger.XForm', null=False, related_name='instances', - on_delete=models.CASCADE) - survey_type = models.ForeignKey( - 'logger.SurveyType', on_delete=models.PROTECT) + "logger.XForm", null=False, related_name="instances", on_delete=models.CASCADE + ) + survey_type = models.ForeignKey("logger.SurveyType", on_delete=models.PROTECT) # shows when we first received this instance date_created = models.DateTimeField(auto_now_add=True) @@ -510,8 +585,9 @@ class Instance(models.Model, InstanceBaseClass): # this will end up representing "date instance was deleted" deleted_at = models.DateTimeField(null=True, default=None) - deleted_by = models.ForeignKey(User, related_name='deleted_instances', - null=True, on_delete=models.SET_NULL) + deleted_by = models.ForeignKey( + User, related_name="deleted_instances", null=True, on_delete=models.SET_NULL + ) # this will be edited when we need to create a new InstanceHistory object last_edited = models.DateTimeField(null=True, default=None) @@ -521,38 +597,36 @@ class Instance(models.Model, InstanceBaseClass): # we add the following additional statuses: # - submitted_via_web # - imported_via_csv - status = models.CharField(max_length=20, - default=u'submitted_via_web') - uuid = models.CharField(max_length=249, default=u'', db_index=True) + status = models.CharField(max_length=20, default="submitted_via_web") + uuid = models.CharField(max_length=249, default="", db_index=True) version = models.CharField(max_length=XFORM_TITLE_LENGTH, null=True) # store a geographic objects associated with this instance geom = models.GeometryCollectionField(null=True) # Keep track of whether all media attachments have been received - media_all_received = models.NullBooleanField( - _("Received All Media Attachemts"), - null=True, - default=True) - total_media = models.PositiveIntegerField(_("Total Media Attachments"), - null=True, - default=0) - media_count = models.PositiveIntegerField(_("Received Media Attachments"), - null=True, - default=0) - checksum = models.CharField(max_length=64, null=True, blank=True, - db_index=True) + media_all_received = models.BooleanField( + _("Received All Media Attachemts"), null=True, default=True + ) + total_media = models.PositiveIntegerField( + _("Total Media Attachments"), null=True, default=0 + ) + media_count = models.PositiveIntegerField( + _("Received Media Attachments"), null=True, default=0 + ) + checksum = models.CharField(max_length=64, null=True, blank=True, db_index=True) # Keep track of submission reviews, only query reviews if true has_a_review = models.BooleanField(_("has_a_review"), default=False) tags = TaggableManager() class Meta: - app_label = 'logger' - unique_together = ('xform', 'uuid') + app_label = "logger" + unique_together = ("xform", "uuid") @classmethod def set_deleted_at(cls, instance_id, deleted_at=timezone.now(), user=None): + """Set's the timestamp when a submission was deleted.""" try: instance = cls.objects.get(id=instance_id) except cls.DoesNotExist: @@ -582,21 +656,22 @@ def get_expected_media(self): """ Returns a list of expected media files from the submission data. """ - if not hasattr(self, '_expected_media'): + if not hasattr(self, "_expected_media"): # pylint: disable=no-member data = self.get_dict() media_list = [] - if 'encryptedXmlFile' in data and self.xform.encrypted: - media_list.append(data['encryptedXmlFile']) - if 'media' in data: + if "encryptedXmlFile" in data and self.xform.encrypted: + media_list.append(data["encryptedXmlFile"]) + if "media" in data: # pylint: disable=no-member - media_list.extend([i['media/file'] for i in data['media']]) + media_list.extend([i["media/file"] for i in data["media"]]) else: - media_xpaths = (self.xform.get_media_survey_xpaths() + - self.xform.get_osm_survey_xpaths()) + media_xpaths = ( + self.xform.get_media_survey_xpaths() + + self.xform.get_osm_survey_xpaths() + ) for media_xpath in media_xpaths: - media_list.extend( - get_values_matching_key(data, media_xpath)) + media_list.extend(get_values_matching_key(data, media_xpath)) # pylint: disable=attribute-defined-outside-init self._expected_media = list(set(media_list)) @@ -607,7 +682,7 @@ def num_of_media(self): """ Returns number of media attachments expected in the submission. """ - if not hasattr(self, '_num_of_media'): + if not hasattr(self, "_num_of_media"): # pylint: disable=attribute-defined-outside-init self._num_of_media = len(self.get_expected_media()) @@ -615,15 +690,20 @@ def num_of_media(self): @property def attachments_count(self): - return self.attachments.filter( - name__in=self.get_expected_media() - ).distinct('name').order_by('name').count() - + """Returns the number of attachments a submission has.""" + return ( + self.attachments.filter(name__in=self.get_expected_media()) + .distinct("name") + .order_by("name") + .count() + ) + + # pylint: disable=arguments-differ def save(self, *args, **kwargs): - force = kwargs.get('force') + force = kwargs.get("force") if force: - del kwargs['force'] + del kwargs["force"] self._check_is_merged_dataset() self._check_active(force) @@ -634,10 +714,11 @@ def save(self, *args, **kwargs): # pylint: disable=no-member self.version = self.json.get(VERSION, self.xform.version) - super(Instance, self).save(*args, **kwargs) + super().save(*args, **kwargs) # pylint: disable=no-member def set_deleted(self, deleted_at=timezone.now(), user=None): + """Set the timestamp and user when a submission is deleted.""" if user: self.deleted_by = user self.deleted_at = deleted_at @@ -650,19 +731,25 @@ def soft_delete_attachments(self, user=None): """ Soft deletes an attachment by adding a deleted_at timestamp. """ - queryset = self.attachments.filter( - ~Q(name__in=self.get_expected_media())) - kwargs = {'deleted_at': timezone.now()} + queryset = self.attachments.filter(~Q(name__in=self.get_expected_media())) + kwargs = {"deleted_at": timezone.now()} if user: - kwargs.update({'deleted_by': user}) + kwargs.update({"deleted_by": user}) queryset.update(**kwargs) +# pylint: disable=unused-argument def post_save_submission(sender, instance=None, created=False, **kwargs): + """Update XForm, Project, JSON field + + - XForm submission coun + - Project date modified + - Update the submission JSON field data + """ if instance.deleted_at is not None: - _update_submission_count_for_today(instance.xform_id, - incr=False, - date_created=instance.date_created) + _update_submission_count_for_today( + instance.xform_id, incr=False, date_created=instance.date_created + ) if ASYNC_POST_SUBMISSION_PROCESSING_ENABLED: update_xform_submission_count.apply_async(args=[instance.pk, created]) @@ -675,25 +762,31 @@ def post_save_submission(sender, instance=None, created=False, **kwargs): update_project_date_modified(instance.pk, created) -post_save.connect(post_save_submission, sender=Instance, - dispatch_uid='post_save_submission') +post_save.connect( + post_save_submission, sender=Instance, dispatch_uid="post_save_submission" +) -post_delete.connect(update_xform_submission_count_delete, sender=Instance, - dispatch_uid='update_xform_submission_count_delete') +post_delete.connect( + update_xform_submission_count_delete, + sender=Instance, + dispatch_uid="update_xform_submission_count_delete", +) class InstanceHistory(models.Model, InstanceBaseClass): + """Stores deleted submission XML to maintain a history of edits.""" class Meta: - app_label = 'logger' + app_label = "logger" xform_instance = models.ForeignKey( - Instance, related_name='submission_history', on_delete=models.CASCADE) + Instance, related_name="submission_history", on_delete=models.CASCADE + ) user = models.ForeignKey(User, null=True, on_delete=models.CASCADE) xml = models.TextField() # old instance id - uuid = models.CharField(max_length=249, default=u'') + uuid = models.CharField(max_length=249, default="") date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) @@ -703,66 +796,81 @@ class Meta: @property def xform(self): + """Returns the XForm object linked to this submission.""" return self.xform_instance.xform @property def attachments(self): + """Returns the attachments linked to this submission.""" return self.xform_instance.attachments.all() @property def json(self): + """Returns the XML submission as a python dictionary object.""" return self.get_full_dict(load_existing=False) @property def status(self): + """Returns the submission's status""" return self.xform_instance.status @property def tags(self): + """Returns the tags linked to the submission.""" return self.xform_instance.tags @property def notes(self): + """Returns the notes attached to the submission.""" return self.xform_instance.notes.all() @property def reviews(self): + """Returns the submission reviews.""" return self.xform_instance.reviews.all() @property def version(self): + """Returns the XForm verison for the submission.""" return self.xform_instance.version @property def osm_data(self): + """Returns the OSM data for the submission.""" return self.xform_instance.osm_data @property def deleted_at(self): + """Mutes the deleted_at method for the history record.""" return None @property def total_media(self): + """Returns the number of attachments linked to submission.""" return self.xform_instance.total_media @property def has_a_review(self): + """Returns the value of a submission.has_a_review.""" return self.xform_instance.has_a_review @property def media_count(self): + """Returns the number of media attached to the submission.""" return self.xform_instance.media_count @property def media_all_received(self): + """Returns the value of the submission.media_all_received.""" return self.xform_instance.media_all_received def _set_parser(self): if not hasattr(self, "_parser"): - self._parser = XFormInstanceParser( - self.xml, self.xform_instance.xform - ) + # pylint: disable=attribute-defined-outside-init + self._parser = XFormInstanceParser(self.xml, self.xform_instance.xform) + # pylint: disable=unused-argument @classmethod def set_deleted_at(cls, instance_id, deleted_at=timezone.now()): + """Mutes the set_deleted_at method for the history record.""" return None diff --git a/onadata/apps/logger/models/merged_xform.py b/onadata/apps/logger/models/merged_xform.py index 26e6c05145..188a37bc9b 100644 --- a/onadata/apps/logger/models/merged_xform.py +++ b/onadata/apps/logger/models/merged_xform.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +MergedXForm model - stores info on forms to merge. +""" from django.db import models from django.db.models.signals import post_save @@ -10,20 +14,24 @@ class MergedXForm(XForm): Merged XForms """ - xforms = models.ManyToManyField( - 'logger.XForm', related_name='mergedxform_ptr') + xforms = models.ManyToManyField("logger.XForm", related_name="mergedxform_ptr") class Meta: - app_label = 'logger' + app_label = "logger" def save(self, *args, **kwargs): set_uuid(self) - return super(MergedXForm, self).save(*args, **kwargs) + return super().save(*args, **kwargs) +# pylint: disable=unused-argument def set_object_permissions(sender, instance=None, created=False, **kwargs): + """Set object permissions when a MergedXForm has been created.""" + if created: + # pylint: disable=import-outside-toplevel from onadata.libs.permissions import OwnerRole + OwnerRole.add(instance.user, instance) OwnerRole.add(instance.user, instance.xform_ptr) @@ -32,6 +40,7 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): OwnerRole.add(instance.created_by, instance.xform_ptr) from onadata.libs.utils.project_utils import set_project_perms_to_xform + set_project_perms_to_xform(instance, instance.project) set_project_perms_to_xform(instance.xform_ptr, instance.project) @@ -39,4 +48,5 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): post_save.connect( set_object_permissions, sender=MergedXForm, - dispatch_uid='set_project_perms_to_merged_xform') + dispatch_uid="set_project_perms_to_merged_xform", +) diff --git a/onadata/apps/logger/models/note.py b/onadata/apps/logger/models/note.py index c7ef49da83..549ea41b3b 100644 --- a/onadata/apps/logger/models/note.py +++ b/onadata/apps/logger/models/note.py @@ -12,12 +12,15 @@ class Note(models.Model): """ Note Model Class """ + note = models.TextField() instance = models.ForeignKey( - 'logger.Instance', related_name='notes', on_delete=models.CASCADE) + "logger.Instance", related_name="notes", on_delete=models.CASCADE + ) instance_field = models.TextField(null=True, blank=True) - created_by = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, - blank=True, on_delete=models.CASCADE) + created_by = models.ForeignKey( + settings.AUTH_USER_MODEL, null=True, blank=True, on_delete=models.CASCADE + ) date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) @@ -25,7 +28,8 @@ class Meta: """ Meta Options for Note Model """ - app_label = 'logger' + + app_label = "logger" def get_data(self): """ @@ -41,5 +45,5 @@ def get_data(self): "owner": owner, "note": self.note, "instance_field": self.instance_field, - "created_by": created_by_id + "created_by": created_by_id, } diff --git a/onadata/apps/logger/models/open_data.py b/onadata/apps/logger/models/open_data.py index 742393c3a2..8c57f717c1 100644 --- a/onadata/apps/logger/models/open_data.py +++ b/onadata/apps/logger/models/open_data.py @@ -7,22 +7,21 @@ from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.contrib.gis.db import models -from django.utils.encoding import python_2_unicode_compatible from onadata.libs.utils.common_tools import get_uuid -@python_2_unicode_compatible class OpenData(models.Model): """ OpenData model represents a way to access private datasets without authentication using the unique uuid. """ + name = models.CharField(max_length=255) uuid = models.CharField(max_length=32, default=get_uuid, unique=True) content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) object_id = models.PositiveIntegerField(null=True, blank=True) - content_object = GenericForeignKey('content_type', 'object_id') + content_object = GenericForeignKey("content_type", "object_id") active = models.BooleanField(default=True) date_created = models.DateTimeField(auto_now_add=True) @@ -32,7 +31,7 @@ def __str__(self): return getattr(self, "name", "") class Meta: - app_label = 'logger' + app_label = "logger" def get_or_create_opendata(xform): @@ -47,8 +46,8 @@ def get_or_create_opendata(xform): return OpenData.objects.get_or_create( object_id=xform.id, defaults={ - 'name': xform.id_string, - 'content_type': content_type, - 'content_object': xform, - } + "name": xform.id_string, + "content_type": content_type, + "content_object": xform, + }, ) diff --git a/onadata/apps/logger/models/osmdata.py b/onadata/apps/logger/models/osmdata.py index 9cc0ec7ce9..b269692bec 100644 --- a/onadata/apps/logger/models/osmdata.py +++ b/onadata/apps/logger/models/osmdata.py @@ -3,59 +3,73 @@ OSM Data model class """ from django.contrib.gis.db import models -from django.contrib.postgres.fields import JSONField class OsmData(models.Model): """ OSM Data information from a submission instance """ + instance = models.ForeignKey( - 'logger.Instance', related_name='osm_data', on_delete=models.CASCADE) + "logger.Instance", related_name="osm_data", on_delete=models.CASCADE + ) xml = models.TextField() osm_id = models.CharField(max_length=20) - osm_type = models.CharField(max_length=10, default='way') - tags = JSONField(default=dict, null=False) + osm_type = models.CharField(max_length=10, default="way") + tags = models.JSONField(default=dict, null=False) geom = models.GeometryCollectionField() filename = models.CharField(max_length=255) - field_name = models.CharField(max_length=255, blank=True, default='') + field_name = models.CharField(max_length=255, blank=True, default="") date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) deleted_at = models.DateTimeField(null=True, default=None) class Meta: - app_label = 'logger' - unique_together = ('instance', 'field_name') + app_label = "logger" + unique_together = ("instance", "field_name") @classmethod def get_tag_keys(cls, xform, field_path, include_prefix=False): + """ + Returns sorted tag keys. + """ query = OsmData.objects.raw( - '''SELECT DISTINCT JSONB_OBJECT_KEYS(tags) as id FROM "logger_osmdata" INNER JOIN "logger_instance" ON ( "logger_osmdata"."instance_id" = "logger_instance"."id" ) WHERE "logger_instance"."xform_id" = %s AND field_name = %s''', # noqa - [xform.pk, field_path] + 'SELECT DISTINCT JSONB_OBJECT_KEYS(tags) as id FROM "logger_osmdata"' + ' INNER JOIN "logger_instance"' + ' ON ( "logger_osmdata"."instance_id" = "logger_instance"."id" )' + ' WHERE "logger_instance"."xform_id" = %s AND field_name = %s', + [xform.pk, field_path], ) - prefix = field_path + u':' if include_prefix else u'' + prefix = field_path + ":" if include_prefix else "" return sorted([prefix + key.id for key in query]) def get_tags_with_prefix(self): - doc = { - self.field_name + ':' + self.osm_type + ':id': self.osm_id - } + """ + Returns tags prefixed by the field_name. + """ + doc = {self.field_name + ":" + self.osm_type + ":id": self.osm_id} for k, v in self.tags.items(): - doc[self.field_name + ':' + k] = v + doc[self.field_name + ":" + k] = v return doc def _set_centroid_in_tags(self): self.tags = self.tags if isinstance(self.tags, dict) else {} if self.geom is not None: - # pylint: disable=E1101 - self.tags.update({ - "ctr:lon": self.geom.centroid.x, - "ctr:lat": self.geom.centroid.y, - }) + # pylint: disable=no-member + self.tags.update( + { + "ctr:lon": self.geom.centroid.x, + "ctr:lat": self.geom.centroid.y, + } + ) - def save(self, *args, **kwargs): + def save( + self, force_insert=False, force_update=False, using=None, update_fields=None + ): self._set_centroid_in_tags() - super(OsmData, self).save(*args, **kwargs) + super().save( + force_insert=False, force_update=False, using=None, update_fields=None + ) diff --git a/onadata/apps/logger/models/project.py b/onadata/apps/logger/models/project.py index 6865da516d..3dc7703c64 100644 --- a/onadata/apps/logger/models/project.py +++ b/onadata/apps/logger/models/project.py @@ -3,14 +3,12 @@ Project model class """ from django.conf import settings -from django.contrib.auth.models import User -from django.contrib.postgres.fields import JSONField +from django.contrib.auth import get_user_model from django.core.exceptions import ValidationError from django.db import models, transaction from django.db.models import Prefetch from django.db.models.signals import post_save from django.utils import timezone -from django.utils.encoding import python_2_unicode_compatible from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from guardian.shortcuts import assign_perm, get_perms_for_model @@ -19,99 +17,139 @@ from onadata.libs.models.base_model import BaseModel from onadata.libs.utils.common_tags import OWNER_TEAM_NAME +# pylint: disable=invalid-name +User = get_user_model() + +# pylint: disable=too-few-public-methods class PrefetchManager(models.Manager): + """Project prefetched manager - prefetches models related to the Project model.""" + def get_queryset(self): - from onadata.apps.logger.models.xform import XForm + """Return a queryset with the XForm, Team, tags, and other related relations + prefetched.""" + # pylint: disable=import-outside-toplevel from onadata.apps.api.models.team import Team - return super(PrefetchManager, self).get_queryset().select_related( - 'created_by', 'organization' - ).prefetch_related( - Prefetch('xform_set', - queryset=XForm.objects.filter(deleted_at__isnull=True) - .select_related('user') - .prefetch_related('user') - .prefetch_related('dataview_set') - .prefetch_related('metadata_set') - .only('id', 'user', 'project', 'title', 'date_created', - 'last_submission_time', 'num_of_submissions', - 'downloadable', 'id_string', 'is_merged_dataset'), - to_attr='xforms_prefetch') - ).prefetch_related('tags')\ - .prefetch_related(Prefetch( - 'projectuserobjectpermission_set', - queryset=ProjectUserObjectPermission.objects.select_related( - 'user__profile__organizationprofile', - 'permission' + from onadata.apps.logger.models.xform import XForm + + # pylint: disable=no-member + return ( + super() + .get_queryset() + .select_related("created_by", "organization") + .prefetch_related( + Prefetch( + "xform_set", + queryset=XForm.objects.filter(deleted_at__isnull=True) + .select_related("user") + .prefetch_related("user") + .prefetch_related("dataview_set") + .prefetch_related("metadata_set") + .only( + "id", + "user", + "project", + "title", + "date_created", + "last_submission_time", + "num_of_submissions", + "downloadable", + "id_string", + "is_merged_dataset", + ), + to_attr="xforms_prefetch", ) - ))\ - .prefetch_related(Prefetch( - 'projectgroupobjectpermission_set', - queryset=ProjectGroupObjectPermission.objects.select_related( - 'group', - 'permission' + ) + .prefetch_related("tags") + .prefetch_related( + Prefetch( + "projectuserobjectpermission_set", + queryset=ProjectUserObjectPermission.objects.select_related( + "user__profile__organizationprofile", "permission" + ), ) - )).prefetch_related('user_stars')\ - .prefetch_related(Prefetch( - 'organization__team_set', - queryset=Team.objects.all().prefetch_related('user_set') - )) + ) + .prefetch_related( + Prefetch( + "projectgroupobjectpermission_set", + queryset=ProjectGroupObjectPermission.objects.select_related( + "group", "permission" + ), + ) + ) + .prefetch_related("user_stars") + .prefetch_related( + Prefetch( + "organization__team_set", + queryset=Team.objects.all().prefetch_related("user_set"), + ) + ) + ) -@python_2_unicode_compatible class Project(BaseModel): """ Project model class """ name = models.CharField(max_length=255) - metadata = JSONField(default=dict) + # pylint: disable=no-member + metadata = models.JSONField(default=dict) organization = models.ForeignKey( - settings.AUTH_USER_MODEL, related_name='project_org', - on_delete=models.CASCADE) + settings.AUTH_USER_MODEL, related_name="project_org", on_delete=models.CASCADE + ) created_by = models.ForeignKey( - settings.AUTH_USER_MODEL, related_name='project_owner', - on_delete=models.CASCADE) - user_stars = models.ManyToManyField(settings.AUTH_USER_MODEL, - related_name='project_stars') + settings.AUTH_USER_MODEL, related_name="project_owner", on_delete=models.CASCADE + ) + user_stars = models.ManyToManyField( + settings.AUTH_USER_MODEL, related_name="project_stars" + ) shared = models.BooleanField(default=False) date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) deleted_at = models.DateTimeField(blank=True, null=True) - deleted_by = models.ForeignKey(User, related_name='project_deleted_by', - blank=True, null=True, default=None, - on_delete=models.SET_NULL) + deleted_by = models.ForeignKey( + User, + related_name="project_deleted_by", + blank=True, + null=True, + default=None, + on_delete=models.SET_NULL, + ) objects = models.Manager() - tags = TaggableManager(related_name='project_tags') + tags = TaggableManager(related_name="project_tags") prefetched = PrefetchManager() class Meta: - app_label = 'logger' - unique_together = (('name', 'organization'),) + app_label = "logger" + unique_together = (("name", "organization"),) permissions = ( - ('add_project_xform', "Can add xform to project"), + ("add_project_xform", "Can add xform to project"), ("report_project_xform", "Can make submissions to the project"), - ('transfer_project', "Can transfer project to different owner"), - ('can_export_project_data', "Can export data in project"), + ("transfer_project", "Can transfer project to different owner"), + ("can_export_project_data", "Can export data in project"), ("view_project_all", "Can view all associated data"), ("view_project_data", "Can view submitted data"), ) def __str__(self): - return u'%s|%s' % (self.organization, self.name) + return f"{self.organization}|{self.name}" def clean(self): - # pylint: disable=E1101 - query_set = Project.objects.exclude(pk=self.pk)\ - .filter(name__iexact=self.name, organization=self.organization) + """Raises a validation error if a project with same name and organization exists.""" + query_set = Project.objects.exclude(pk=self.pk).filter( + name__iexact=self.name, organization=self.organization + ) if query_set.exists(): - raise ValidationError(u'Project name "%s" is already in' - u' use in this account.' - % self.name.lower()) + raise ValidationError( + f'Project name "{self.name.lower()}" is already in' + " use in this account." + ) @property def user(self): + """Returns the user who created the project.""" return self.created_by @transaction.atomic() @@ -124,7 +162,7 @@ def soft_delete(self, user=None): """ soft_deletion_time = timezone.now() - deletion_suffix = soft_deletion_time.strftime('-deleted-at-%s') + deletion_suffix = soft_deletion_time.strftime("-deleted-at-%s") self.deleted_at = soft_deletion_time self.name += deletion_suffix if user is not None: @@ -135,14 +173,17 @@ def soft_delete(self, user=None): form.soft_delete(user=user) +# pylint: disable=unused-argument def set_object_permissions(sender, instance=None, created=False, **kwargs): + """Sets permissions to users who are owners of the organization.""" if created: for perm in get_perms_for_model(Project): assign_perm(perm.codename, instance.organization, instance) - owners = instance.organization.team_set\ - .filter(name="{}#{}".format(instance.organization.username, - OWNER_TEAM_NAME), organization=instance.organization) + owners = instance.organization.team_set.filter( + name=f"{instance.organization.username}#{OWNER_TEAM_NAME}", + organization=instance.organization, + ) for owner in owners: assign_perm(perm.codename, owner, instance) @@ -153,16 +194,21 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): assign_perm(perm.codename, instance.created_by, instance) -post_save.connect(set_object_permissions, sender=Project, - dispatch_uid='set_project_object_permissions') +post_save.connect( + set_object_permissions, + sender=Project, + dispatch_uid="set_project_object_permissions", +) +# pylint: disable=too-few-public-methods class ProjectUserObjectPermission(UserObjectPermissionBase): """Guardian model to create direct foreign keys.""" content_object = models.ForeignKey(Project, on_delete=models.CASCADE) +# pylint: disable=too-few-public-methods class ProjectGroupObjectPermission(GroupObjectPermissionBase): """Guardian model to create direct foreign keys.""" diff --git a/onadata/apps/logger/models/submission_review.py b/onadata/apps/logger/models/submission_review.py index 34aac1a413..7a8205130e 100644 --- a/onadata/apps/logger/models/submission_review.py +++ b/onadata/apps/logger/models/submission_review.py @@ -8,7 +8,7 @@ from django.db import models from django.db.models.signals import post_save from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ def update_instance_json_on_save(sender, instance, **kwargs): @@ -26,40 +26,44 @@ class SubmissionReview(models.Model): SubmissionReview Model Class """ - APPROVED = '1' - REJECTED = '2' - PENDING = '3' + APPROVED = "1" + REJECTED = "2" + PENDING = "3" - STATUS_CHOICES = ((APPROVED, _('Approved')), (PENDING, _('Pending')), - (REJECTED, _('Rejected'))) + STATUS_CHOICES = ( + (APPROVED, _("Approved")), + (PENDING, _("Pending")), + (REJECTED, _("Rejected")), + ) instance = models.ForeignKey( - 'logger.Instance', related_name='reviews', on_delete=models.CASCADE) + "logger.Instance", related_name="reviews", on_delete=models.CASCADE + ) note = models.ForeignKey( - 'logger.Note', - related_name='notes', + "logger.Note", + related_name="notes", blank=True, null=True, default=None, - on_delete=models.SET_NULL) + on_delete=models.SET_NULL, + ) created_by = models.ForeignKey( settings.AUTH_USER_MODEL, null=True, blank=True, default=None, - on_delete=models.CASCADE) + on_delete=models.CASCADE, + ) status = models.CharField( - 'Status', - max_length=1, - choices=STATUS_CHOICES, - default=PENDING, - db_index=True) + "Status", max_length=1, choices=STATUS_CHOICES, default=PENDING, db_index=True + ) deleted_at = models.DateTimeField(null=True, default=None, db_index=True) deleted_by = models.ForeignKey( settings.AUTH_USER_MODEL, - related_name='deleted_reviews', + related_name="deleted_reviews", null=True, - on_delete=models.SET_NULL) + on_delete=models.SET_NULL, + ) date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) @@ -67,7 +71,8 @@ class Meta: """ Meta Options for SubmissionReview """ - app_label = 'logger' + + app_label = "logger" def get_note_text(self): """ @@ -90,5 +95,7 @@ def set_deleted(self, deleted_at=timezone.now(), user=None): post_save.connect( - update_instance_json_on_save, sender=SubmissionReview, - dispatch_uid='update_instance_json_on_save') + update_instance_json_on_save, + sender=SubmissionReview, + dispatch_uid="update_instance_json_on_save", +) diff --git a/onadata/apps/logger/models/survey_type.py b/onadata/apps/logger/models/survey_type.py index 6201c85b7d..5a25fe0547 100644 --- a/onadata/apps/logger/models/survey_type.py +++ b/onadata/apps/logger/models/survey_type.py @@ -3,18 +3,17 @@ Survey type model class """ from django.db import models -from django.utils.encoding import python_2_unicode_compatible -@python_2_unicode_compatible class SurveyType(models.Model): """ Survey type model class """ + slug = models.CharField(max_length=100, unique=True) class Meta: - app_label = 'logger' + app_label = "logger" def __str__(self): return "SurveyType: %s" % self.slug diff --git a/onadata/apps/logger/models/widget.py b/onadata/apps/logger/models/widget.py index cfb48fb7ff..44e878afac 100644 --- a/onadata/apps/logger/models/widget.py +++ b/onadata/apps/logger/models/widget.py @@ -1,10 +1,9 @@ from builtins import str as text -from past.builtins import basestring +from django.db.models import JSONField from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.contrib.gis.db import models -from django.contrib.postgres.fields import JSONField from ordered_model.models import OrderedModel from querybuilder.fields import AvgField, CountField, SimpleField, SumField from querybuilder.query import Query @@ -12,50 +11,46 @@ from onadata.apps.logger.models.data_view import DataView from onadata.apps.logger.models.instance import Instance from onadata.apps.logger.models.xform import XForm -from onadata.libs.utils.chart_tools import (DATA_TYPE_MAP, - _flatten_multiple_dict_into_one, - _use_labels_from_group_by_name, - get_field_choices, - get_field_from_field_xpath, - get_field_label) -from onadata.libs.utils.common_tags import (NUMERIC_LIST, SELECT_ONE, - SUBMISSION_TIME) +from onadata.libs.utils.chart_tools import ( + DATA_TYPE_MAP, + _flatten_multiple_dict_into_one, + _use_labels_from_group_by_name, + get_field_choices, + get_field_from_field_xpath, + get_field_label, +) +from onadata.libs.utils.common_tags import NUMERIC_LIST, SELECT_ONE, SUBMISSION_TIME from onadata.libs.utils.common_tools import get_uuid class Widget(OrderedModel): - CHARTS = 'charts' + CHARTS = "charts" # Other widgets types to be added later - WIDGETS_TYPES = ((CHARTS, 'Charts'), ) + WIDGETS_TYPES = ((CHARTS, "Charts"),) # Will hold either XForm or DataView Model content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) object_id = models.PositiveIntegerField() - content_object = GenericForeignKey('content_type', 'object_id') + content_object = GenericForeignKey("content_type", "object_id") - widget_type = models.CharField( - max_length=25, choices=WIDGETS_TYPES, default=CHARTS) + widget_type = models.CharField(max_length=25, choices=WIDGETS_TYPES, default=CHARTS) view_type = models.CharField(max_length=50) column = models.CharField(max_length=255) - group_by = models.CharField( - null=True, default=None, max_length=255, blank=True) - - title = models.CharField( - null=True, default=None, max_length=255, blank=True) - description = models.CharField( - null=True, default=None, max_length=255, blank=True) - aggregation = models.CharField( - null=True, default=None, max_length=255, blank=True) + group_by = models.CharField(null=True, default=None, max_length=255, blank=True) + + title = models.CharField(null=True, default=None, max_length=255, blank=True) + description = models.CharField(null=True, default=None, max_length=255, blank=True) + aggregation = models.CharField(null=True, default=None, max_length=255, blank=True) key = models.CharField(db_index=True, unique=True, max_length=32) date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) - order_with_respect_to = 'content_type' + order_with_respect_to = "content_type" metadata = JSONField(default=dict, blank=True) class Meta(OrderedModel.Meta): - app_label = 'logger' + app_label = "logger" def save(self, *args, **kwargs): @@ -77,61 +72,60 @@ def query_data(cls, widget): field = get_field_from_field_xpath(column, xform) - if isinstance(field, basestring) and field == SUBMISSION_TIME: - field_label = 'Submission Time' - field_xpath = '_submission_time' - field_type = 'datetime' - data_type = DATA_TYPE_MAP.get(field_type, 'categorized') + if isinstance(field, str) and field == SUBMISSION_TIME: + field_label = "Submission Time" + field_xpath = "_submission_time" + field_type = "datetime" + data_type = DATA_TYPE_MAP.get(field_type, "categorized") else: field_type = field.type - data_type = DATA_TYPE_MAP.get(field.type, 'categorized') + data_type = DATA_TYPE_MAP.get(field.type, "categorized") field_xpath = field.get_abbreviated_xpath() field_label = get_field_label(field) columns = [ - SimpleField( - field="json->>'%s'" % text(column), - alias='{}'.format(column)), - CountField( - field="json->>'%s'" % text(column), - alias='count') + SimpleField(field="json->>'%s'" % text(column), alias="{}".format(column)), + CountField(field="json->>'%s'" % text(column), alias="count"), ] if group_by: if field_type in NUMERIC_LIST: column_field = SimpleField( - field="json->>'%s'" % text(column), - cast="float", - alias=column) + field="json->>'%s'" % text(column), cast="float", alias=column + ) else: column_field = SimpleField( - field="json->>'%s'" % text(column), alias=column) + field="json->>'%s'" % text(column), alias=column + ) # build inner query - inner_query_columns = \ - [column_field, - SimpleField(field="json->>'%s'" % text(group_by), - alias=group_by), - SimpleField(field="xform_id"), - SimpleField(field="deleted_at")] + inner_query_columns = [ + column_field, + SimpleField(field="json->>'%s'" % text(group_by), alias=group_by), + SimpleField(field="xform_id"), + SimpleField(field="deleted_at"), + ] inner_query = Query().from_table(Instance, inner_query_columns) # build group-by query if field_type in NUMERIC_LIST: columns = [ - SimpleField(field=group_by, alias='%s' % group_by), + SimpleField(field=group_by, alias="%s" % group_by), SumField(field=column, alias="sum"), - AvgField(field=column, alias="mean") + AvgField(field=column, alias="mean"), ] elif field_type == SELECT_ONE: columns = [ - SimpleField(field=column, alias='%s' % column), - SimpleField(field=group_by, alias='%s' % group_by), - CountField(field="*", alias='count') + SimpleField(field=column, alias="%s" % column), + SimpleField(field=group_by, alias="%s" % group_by), + CountField(field="*", alias="count"), ] - query = Query().from_table({'inner_query': inner_query}, columns).\ - where(xform_id=xform.pk, deleted_at=None) + query = ( + Query() + .from_table({"inner_query": inner_query}, columns) + .where(xform_id=xform.pk, deleted_at=None) + ) if field_type == SELECT_ONE: query.group_by(column).group_by(group_by) @@ -139,8 +133,11 @@ def query_data(cls, widget): query.group_by(group_by) else: - query = Query().from_table(Instance, columns).\ - where(xform_id=xform.pk, deleted_at=None) + query = ( + Query() + .from_table(Instance, columns) + .where(xform_id=xform.pk, deleted_at=None) + ) query.group_by("json->>'%s'" % text(column)) # run query @@ -148,14 +145,14 @@ def query_data(cls, widget): # flatten multiple dict if select one with group by if field_type == SELECT_ONE and group_by: - records = _flatten_multiple_dict_into_one(column, group_by, - records) + records = _flatten_multiple_dict_into_one(column, group_by, records) # use labels if group by if group_by: group_by_field = get_field_from_field_xpath(group_by, xform) choices = get_field_choices(group_by, xform) records = _use_labels_from_group_by_name( - group_by, group_by_field, data_type, records, choices=choices) + group_by, group_by_field, data_type, records, choices=choices + ) return { "field_type": field_type, @@ -163,5 +160,5 @@ def query_data(cls, widget): "field_xpath": field_xpath, "field_label": field_label, "grouped_by": group_by, - "data": records + "data": records, } diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index ea8b84296f..a605382ef5 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -1,114 +1,158 @@ +# -*- coding: utf-8 -*- +""" +The XForm model +""" +# pylint: disable=too-many-lines +import hashlib import json import os import re -from builtins import bytes as b, str as text from datetime import datetime -from hashlib import md5 from xml.dom import Node -import pytz from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.contrib.contenttypes.fields import GenericRelation -from django.core.exceptions import ObjectDoesNotExist from django.core.cache import cache +from django.core.exceptions import ObjectDoesNotExist from django.db import models, transaction from django.db.models import Sum -from django.db.models.signals import post_delete, post_save, pre_save +from django.db.models.signals import post_delete, pre_save from django.urls import reverse from django.utils import timezone -from django.utils.encoding import python_2_unicode_compatible -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_lazy -from future.utils import iteritems -from future.utils import listvalues +from django.utils.html import conditional_escape +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy + +import pytz from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase -from past.builtins import cmp -from pyxform import ( - SurveyElementBuilder, constants, create_survey_element_from_dict) +from pyxform import SurveyElementBuilder, constants, create_survey_element_from_dict from pyxform.question import Question from pyxform.section import RepeatingSection from pyxform.xform2json import create_survey_element_from_xml +from six import iteritems from taggit.managers import TaggableManager -from onadata.apps.logger.xform_instance_parser import (XLSFormError, - clean_and_parse_xml) -from django.utils.html import conditional_escape +from onadata.apps.logger.xform_instance_parser import XLSFormError, clean_and_parse_xml from onadata.libs.models.base_model import BaseModel from onadata.libs.utils.cache_tools import ( - IS_ORG, PROJ_BASE_FORMS_CACHE, PROJ_FORMS_CACHE, - PROJ_NUM_DATASET_CACHE, PROJ_SUB_DATE_CACHE, XFORM_COUNT, - PROJ_OWNER_CACHE, XFORM_SUBMISSION_COUNT_FOR_DAY, - XFORM_SUBMISSION_COUNT_FOR_DAY_DATE, safe_delete) -from onadata.libs.utils.common_tags import (DURATION, ID, KNOWN_MEDIA_TYPES, - MEDIA_ALL_RECEIVED, MEDIA_COUNT, - NOTES, SUBMISSION_TIME, - SUBMITTED_BY, TAGS, TOTAL_MEDIA, - UUID, VERSION, REVIEW_STATUS, - REVIEW_COMMENT, - MULTIPLE_SELECT_TYPE, - DATE_MODIFIED) + PROJ_BASE_FORMS_CACHE, + PROJ_FORMS_CACHE, + PROJ_NUM_DATASET_CACHE, + PROJ_OWNER_CACHE, + PROJ_SUB_DATE_CACHE, + XFORM_COUNT, + XFORM_SUBMISSION_COUNT_FOR_DAY, + XFORM_SUBMISSION_COUNT_FOR_DAY_DATE, + safe_delete, +) +from onadata.libs.utils.common_tags import ( + DATE_MODIFIED, + DURATION, + ID, + KNOWN_MEDIA_TYPES, + MEDIA_ALL_RECEIVED, + MEDIA_COUNT, + MULTIPLE_SELECT_TYPE, + NOTES, + REVIEW_COMMENT, + REVIEW_STATUS, + SUBMISSION_TIME, + SUBMITTED_BY, + TAGS, + TOTAL_MEDIA, + UUID, + VERSION, +) from onadata.libs.utils.model_tools import queryset_iterator from onadata.libs.utils.mongo import _encode_for_mongo QUESTION_TYPES_TO_EXCLUDE = [ - u'note', + "note", ] XFORM_TITLE_LENGTH = 255 -title_pattern = re.compile(r"(.*?)") +TITLE_PATTERN = re.compile(r"(.*?)") + +# pylint: disable=invalid-name +User = get_user_model() + + +def cmp(x, y): + """Returns the difference on the comparison of ``x`` and ``y``.""" + return (x > y) - (x < y) def question_types_to_exclude(_type): + """Returns True if ``_type`` is in QUESTION_TYPES_TO_EXCLUDE.""" return _type in QUESTION_TYPES_TO_EXCLUDE def upload_to(instance, filename): - return os.path.join(instance.user.username, 'xls', - os.path.split(filename)[1]) + """Returns the path to upload an XLSForm file to.""" + return os.path.join(instance.user.username, "xls", os.path.split(filename)[1]) -def contains_xml_invalid_char(text, invalids=['&', '>', '<']): +def contains_xml_invalid_char(text, invalids=None): """Check whether 'text' contains ANY invalid xml chars""" + invalids = ["&", ">", "<"] if invalids is None else invalids return 1 in [c in text for c in invalids] -class DictOrganizer(object): +def _additional_headers(): + return [ + "_xform_id_string", + "_percentage_complete", + "_status", + "_attachments", + "_potential_duplicates", + ] + + +class DictOrganizer: + """Adds parent index information in a submission record.""" + def set_dict_iterator(self, dict_iterator): + """Set's the dict iterator.""" + # pylint: disable=attribute-defined-outside-init self._dict_iterator = dict_iterator # Every section will get its own table # I need to think of an easy way to flatten out a dictionary # parent name, index, table name, data - def _build_obs_from_dict(self, d, obs, table_name, parent_table_name, - parent_index): + # pylint: disable=too-many-arguments + def _build_obs_from_dict( + self, dict_item, obs, table_name, parent_table_name, parent_index + ): if table_name not in obs: obs[table_name] = [] this_index = len(obs[table_name]) - obs[table_name].append({ - u"_parent_table_name": parent_table_name, - u"_parent_index": parent_index, - }) - for (k, v) in iteritems(d): + obs[table_name].append( + { + "_parent_table_name": parent_table_name, + "_parent_index": parent_index, + } + ) + for (k, v) in iteritems(dict_item): if isinstance(v, dict) and isinstance(v, list): if k in obs[table_name][-1]: raise AssertionError() obs[table_name][-1][k] = v - obs[table_name][-1][u"_index"] = this_index + obs[table_name][-1]["_index"] = this_index - for (k, v) in iteritems(d): + for (k, v) in iteritems(dict_item): if isinstance(v, dict): kwargs = { - "d": v, + "dict_item": v, "obs": obs, "table_name": k, "parent_table_name": table_name, - "parent_index": this_index + "parent_index": this_index, } self._build_obs_from_dict(**kwargs) elif isinstance(v, list): for item in v: kwargs = { - "d": item, + "dict_item": item, "obs": obs, "table_name": k, "parent_table_name": table_name, @@ -117,15 +161,16 @@ def _build_obs_from_dict(self, d, obs, table_name, parent_table_name, self._build_obs_from_dict(**kwargs) return obs - def get_observation_from_dict(self, d): - if len(list(d)) != 1: + def get_observation_from_dict(self, item): + """Returns a dict that has observations added from ``item``.""" + if len(list(item)) != 1: raise AssertionError() - root_name = list(d)[0] + root_name = list(item)[0] kwargs = { - "d": d[root_name], + "dict_item": item[root_name], "obs": {}, "table_name": root_name, - "parent_table_name": u"", + "parent_table_name": "", "parent_index": -1, } @@ -133,7 +178,7 @@ def get_observation_from_dict(self, d): class DuplicateUUIDError(Exception): - pass + """Exception to raise when there are duplicate UUIDS in an XForm XML.""" def get_forms_shared_with_user(user): @@ -142,10 +187,13 @@ def get_forms_shared_with_user(user): """ xforms = XForm.objects.filter( pk__in=user.xformuserobjectpermission_set.values_list( - 'content_object_id', flat=True).distinct(), - downloadable=True, deleted_at__isnull=True) + "content_object_id", flat=True + ).distinct(), + downloadable=True, + deleted_at__isnull=True, + ) - return xforms.exclude(user=user).select_related('user') + return xforms.exclude(user=user).select_related("user") def check_version_set(survey): @@ -155,124 +203,142 @@ def check_version_set(survey): """ # get the json and check for the version key - survey_json = json.loads(survey.to_json()) + survey_json = survey.to_json_dict() if not survey_json.get("version"): # set utc time as the default version - survey_json['version'] = \ - datetime.utcnow().strftime("%Y%m%d%H%M") + survey_json["version"] = datetime.utcnow().strftime("%Y%m%d%H%M") builder = SurveyElementBuilder() - survey = builder.create_survey_element_from_json( - json.dumps(survey_json)) + if isinstance(survey_json, str): + survey = builder.create_survey_element_from_json(survey_json) + elif isinstance(survey_json, dict): + survey = builder.create_survey_element_from_dict(survey_json) return survey -def _expand_select_all_that_apply(d, key, e): - if e and e.bind.get(u"type") == u"string"\ - and e.type == MULTIPLE_SELECT_TYPE: - options_selected = d[key].split() - for child in e.children: +def _expand_select_all_that_apply(item, key, elem): + """Split's a select multiple into individual keys""" + if elem and elem.bind.get("type") == "string" and elem.type == MULTIPLE_SELECT_TYPE: + options_selected = item[key].split() + for child in elem.children: new_key = child.get_abbreviated_xpath() - d[new_key] = child.name in options_selected + item[new_key] = child.name in options_selected - del d[key] + del item[key] -class XFormMixin(object): +# pylint: disable=too-many-instance-attributes,too-many-public-methods +class XFormMixin: + """XForm mixin class - adds helper functions.""" - GEODATA_SUFFIXES = ['latitude', 'longitude', 'altitude', 'precision'] + GEODATA_SUFFIXES = ["latitude", "longitude", "altitude", "precision"] - PREFIX_NAME_REGEX = re.compile(r'(?P.+/)(?P[^/]+)$') + PREFIX_NAME_REGEX = re.compile(r"(?P.+/)(?P[^/]+)$") - def _set_uuid_in_xml(self, file_name=None): + # pylint: disable=too-many-locals + def set_uuid_in_xml(self, file_name=None): """ Add bind to automatically set UUID node in XML. """ if not file_name: file_name = self.file_name() - file_name, file_ext = os.path.splitext(file_name) + file_name, _file_ext = os.path.splitext(file_name) doc = clean_and_parse_xml(self.xml) model_nodes = doc.getElementsByTagName("model") if len(model_nodes) != 1: - raise Exception(u"xml contains multiple model nodes") + raise Exception("xml contains multiple model nodes") model_node = model_nodes[0] instance_nodes = [ - node for node in model_node.childNodes - if node.nodeType == Node.ELEMENT_NODE and - node.tagName.lower() == "instance" and not node.hasAttribute("id") + node + for node in model_node.childNodes + if node.nodeType == Node.ELEMENT_NODE + and node.tagName.lower() == "instance" + and not node.hasAttribute("id") ] if len(instance_nodes) != 1: - raise Exception(u"Multiple instance nodes without the id " - u"attribute, can't tell which is the main one") + raise Exception( + "Multiple instance nodes without the id " + "attribute, can't tell which is the main one" + ) instance_node = instance_nodes[0] # get the first child whose id attribute matches our id_string survey_nodes = [ - node for node in instance_node.childNodes - if node.nodeType == Node.ELEMENT_NODE and - (node.tagName == file_name or node.attributes.get('id')) + node + for node in instance_node.childNodes + if node.nodeType == Node.ELEMENT_NODE + and (node.tagName == file_name or node.attributes.get("id")) ] if len(survey_nodes) != 1: - raise Exception( - u"Multiple survey nodes with the id '%s'" % self.id_string) + raise Exception("Multiple survey nodes with the id '{self.id_string}'") survey_node = survey_nodes[0] formhub_nodes = [ - n for n in survey_node.childNodes + n + for n in survey_node.childNodes if n.nodeType == Node.ELEMENT_NODE and n.tagName == "formhub" ] if len(formhub_nodes) > 1: - raise Exception( - u"Multiple formhub nodes within main instance node") - elif len(formhub_nodes) == 1: + raise Exception("Multiple formhub nodes within main instance node") + if len(formhub_nodes) == 1: formhub_node = formhub_nodes[0] else: formhub_node = survey_node.insertBefore( - doc.createElement("formhub"), survey_node.firstChild) + doc.createElement("formhub"), survey_node.firstChild + ) uuid_nodes = [ - node for node in formhub_node.childNodes + node + for node in formhub_node.childNodes if node.nodeType == Node.ELEMENT_NODE and node.tagName == "uuid" ] - if len(uuid_nodes) == 0: + if not uuid_nodes: formhub_node.appendChild(doc.createElement("uuid")) - if len(formhub_nodes) == 0: + if not formhub_nodes: # append the calculate bind node calculate_node = doc.createElement("bind") calculate_node.setAttribute( - "nodeset", "/%s/formhub/uuid" % survey_node.tagName) + "nodeset", f"/{survey_node.tagName}/formhub/uuid" + ) calculate_node.setAttribute("type", "string") - calculate_node.setAttribute("calculate", "'%s'" % self.uuid) + calculate_node.setAttribute("calculate", f"'{self.uuid}'") model_node.appendChild(calculate_node) - self.xml = doc.toprettyxml(indent=" ", encoding='utf-8') + self.xml = doc.toprettyxml(indent=" ", encoding="utf-8") # hack # http://ronrothman.com/public/leftbraned/xml-dom-minidom-toprettyxml-\ # and-silly-whitespace/ - text_re = re.compile('(>)\n\s*(\s[^<>\s].*?)\n\s*(\s)\n( )*') - pretty_xml = text_re.sub(lambda m: ''.join(m.group(1, 2, 3)), - self.xml.decode('utf-8')) - inline_output = output_re.sub('\g<1>', pretty_xml) # noqa - inline_output = re.compile('').sub( # noqa - '', inline_output) + text_re = re.compile(r"(>)\n\s*(\s[^<>\s].*?)\n\s*(\s)\n( )*") + pretty_xml = text_re.sub( + lambda m: "".join(m.group(1, 2, 3)), self.xml.decode("utf-8") + ) + inline_output = output_re.sub("\g<1>", pretty_xml) # noqa + inline_output = re.compile(r"").sub( + "", inline_output + ) self.xml = inline_output + # pylint: disable=too-few-public-methods class Meta: + """A proxy Meta class""" + app_label = "viewer" proxy = True @property def has_id_string_changed(self): - return getattr(self, '_id_string_changed', False) + """Returns the boolean value of `_id_string_changed`.""" + return getattr(self, "_id_string_changed", False) def add_instances(self): + """Returns all instances as a list of python objects.""" _get_observation_from_dict = DictOrganizer().get_observation_from_dict return [ @@ -289,34 +355,39 @@ def _id_string_already_exists_in_account(self, id_string): return True def get_unique_id_string(self, id_string, count=0): + """Checks and returns a unique ``id_string``.""" # used to generate a new id_string for new data_dictionary object if # id_string already existed if self._id_string_already_exists_in_account(id_string): if count != 0: - if re.match(r'\w+_\d+$', id_string): - a = id_string.split('_') - id_string = "_".join(a[:-1]) + if re.match(r"\w+_\d+$", id_string): + parts = id_string.split("_") + id_string = "_".join(parts[:-1]) count += 1 - id_string = "{}_{}".format(id_string, count) + id_string = f"{id_string}_{count}" return self.get_unique_id_string(id_string, count) return id_string def get_survey(self): + """Returns an XML XForm survey object.""" if not hasattr(self, "_survey"): try: builder = SurveyElementBuilder() - self._survey = \ - builder.create_survey_element_from_json(self.json) + if isinstance(self.json, str): + self._survey = builder.create_survey_element_from_json(self.json) + if isinstance(self.json, dict): + self._survey = builder.create_survey_element_from_dict(self.json) except ValueError: - xml = b(bytearray(self.xml, encoding='utf-8')) + xml = bytes(bytearray(self.xml, encoding="utf-8")) self._survey = create_survey_element_from_xml(xml) return self._survey survey = property(get_survey) def get_survey_elements(self): + """Returns an iterator of all survey elements.""" return self.survey.iter_descendants() def get_survey_element(self, name_or_xpath): @@ -331,11 +402,10 @@ def get_survey_element(self, name_or_xpath): # search by name if xpath fails fields = [ - field for field in self.get_survey_elements() - if field.name == name_or_xpath + field for field in self.get_survey_elements() if field.name == name_or_xpath ] - return fields[0] if len(fields) else None + return fields[0] if fields else None def get_child_elements(self, name_or_xpath, split_select_multiples=True): """Returns a list of survey elements children in a flat list. @@ -343,16 +413,18 @@ def get_child_elements(self, name_or_xpath, split_select_multiples=True): appended to the list. If the name_or_xpath is a repeat we iterate through the child elements as well. """ - GROUP_AND_SELECT_MULTIPLES = ['group'] + group_and_select_multiples = ["group"] if split_select_multiples: - GROUP_AND_SELECT_MULTIPLES += ['select all that apply'] + group_and_select_multiples += ["select all that apply"] - def flatten(elem, items=[]): + def flatten(elem, items=None): + items = [] if items is None else items results = [] if elem: xpath = elem.get_abbreviated_xpath() - if elem.type in GROUP_AND_SELECT_MULTIPLES or \ - (xpath == name_or_xpath and elem.type == 'repeat'): + if elem.type in group_and_select_multiples or ( + xpath == name_or_xpath and elem.type == "repeat" + ): for child in elem.children: results += flatten(child) else: @@ -364,16 +436,16 @@ def flatten(elem, items=[]): return flatten(element) - def get_choice_label(self, field, choice_value, lang='English'): - choices = [ - choice for choice in field.children if choice.name == choice_value - ] - if len(choices): + # pylint: disable=no-self-use + def get_choice_label(self, field, choice_value, lang="English"): + """Returns a choice's label for the given ``field`` and ``choice_value``.""" + choices = [choice for choice in field.children if choice.name == choice_value] + if choices: choice = choices[0] label = choice.label if isinstance(label, dict): - label = label.get(lang, listvalues(choice.label)[0]) + label = label.get(lang, list(choice.label.values())[0]) return label @@ -386,36 +458,38 @@ def get_mongo_field_names_dict(self): """ names = {} for elem in self.get_survey_elements(): - names[_encode_for_mongo(text(elem.get_abbreviated_xpath()))] = \ - elem.get_abbreviated_xpath() + names[ + _encode_for_mongo(str(elem.get_abbreviated_xpath())) + ] = elem.get_abbreviated_xpath() return names survey_elements = property(get_survey_elements) def get_field_name_xpaths_only(self): + """Returns the abbreviated_xpath of all fields in a survey form.""" return [ - elem.get_abbreviated_xpath() for elem in self.survey_elements - if elem.type != '' and elem.type != 'survey' + elem.get_abbreviated_xpath() + for elem in self.survey_elements + if elem.type not in ("", "survey") ] def geopoint_xpaths(self): + """Returns the abbreviated_xpath of all fields of type `geopoint`.""" survey_elements = self.get_survey_elements() return [ - e.get_abbreviated_xpath() for e in survey_elements - if e.bind.get(u'type') == u'geopoint' + e.get_abbreviated_xpath() + for e in survey_elements + if e.bind.get("type") == "geopoint" ] def xpath_of_first_geopoint(self): + """Returns the abbreviated_xpath of the first field of type `geopoint`.""" geo_xpaths = self.geopoint_xpaths() return len(geo_xpaths) and geo_xpaths[0] - def xpaths(self, - prefix='', - survey_element=None, - result=None, - repeat_iterations=4): + def xpaths(self, prefix="", survey_element=None, result=None, repeat_iterations=4): """ Return a list of XPaths for this survey that will be used as headers for the csv export. @@ -426,13 +500,15 @@ def xpaths(self, return [] result = [] if result is None else result - path = '/'.join([prefix, text(survey_element.name)]) + path = "/".join([prefix, str(survey_element.name)]) if survey_element.children is not None: # add xpaths to result for each child - indices = [''] if not isinstance(survey_element, - RepeatingSection) else \ - ['[%d]' % (i + 1) for i in range(repeat_iterations)] + indices = ( + [""] + if not isinstance(survey_element, RepeatingSection) + else [f"[{(i + 1)}]" for i in range(repeat_iterations)] + ) for i in indices: for e in survey_element.children: self.xpaths(path + i, e, result, repeat_iterations) @@ -442,12 +518,14 @@ def xpaths(self, # replace the single question column with a column for each # item in a select all that apply question. - if survey_element.bind.get(u'type') == u'string' \ - and survey_element.type == MULTIPLE_SELECT_TYPE: + if ( + survey_element.bind.get("type") == "string" + and survey_element.type == MULTIPLE_SELECT_TYPE + ): result.pop() for child in survey_element.children: - result.append('/'.join([path, child.name])) - elif survey_element.bind.get(u'type') == u'geopoint': + result.append("/".join([path, child.name])) + elif survey_element.bind.get("type") == "geopoint": result += self.get_additional_geopoint_xpaths(path) return result @@ -461,74 +539,83 @@ def get_additional_geopoint_xpaths(cls, xpath): DataDictionary.GEODATA_SUFFIXES """ match = cls.PREFIX_NAME_REGEX.match(xpath) - prefix = '' + prefix = "" name = xpath if match: - prefix = match.groupdict()['prefix'] - name = match.groupdict()['name'] - - return [ - '_'.join([prefix, name, suffix]) for suffix in cls.GEODATA_SUFFIXES - ] + prefix = match.groupdict()["prefix"] + name = match.groupdict()["name"] - def _additional_headers(self): - return [ - u'_xform_id_string', u'_percentage_complete', u'_status', - u'_attachments', u'_potential_duplicates' - ] + return ["_".join([prefix, name, suffix]) for suffix in cls.GEODATA_SUFFIXES] - def get_headers( - self, include_additional_headers=False, repeat_iterations=4): + def get_headers(self, include_additional_headers=False, repeat_iterations=4): """ Return a list of headers for a csv file. """ + def shorten(xpath): - xpath_list = xpath.split('/') - return '/'.join(xpath_list[2:]) + """Returns the shortened part of an ``xpath`` removing the root node.""" + xpath_list = xpath.split("/") + return "/".join(xpath_list[2:]) header_list = [ - shorten(xpath) for xpath in self.xpaths( - repeat_iterations=repeat_iterations)] + shorten(xpath) for xpath in self.xpaths(repeat_iterations=repeat_iterations) + ] header_list += [ - ID, UUID, SUBMISSION_TIME, DATE_MODIFIED, TAGS, NOTES, - REVIEW_STATUS, REVIEW_COMMENT, VERSION, DURATION, - SUBMITTED_BY, TOTAL_MEDIA, MEDIA_COUNT, - MEDIA_ALL_RECEIVED + ID, + UUID, + SUBMISSION_TIME, + DATE_MODIFIED, + TAGS, + NOTES, + REVIEW_STATUS, + REVIEW_COMMENT, + VERSION, + DURATION, + SUBMITTED_BY, + TOTAL_MEDIA, + MEDIA_COUNT, + MEDIA_ALL_RECEIVED, ] if include_additional_headers: - header_list += self._additional_headers() + header_list += _additional_headers() return header_list def get_keys(self): + """Return all XForm headers.""" + def remove_first_index(xpath): - return re.sub(r'\[1\]', '', xpath) + """Removes the first index from an ``xpath``.""" + return re.sub(r"\[1\]", "", xpath) return [remove_first_index(header) for header in self.get_headers()] def get_element(self, abbreviated_xpath): + """Returns an XML element""" if not hasattr(self, "_survey_elements"): self._survey_elements = {} for e in self.get_survey_elements(): self._survey_elements[e.get_abbreviated_xpath()] = e def remove_all_indices(xpath): - return re.sub(r"\[\d+\]", u"", xpath) + """Removes all indices from an ``xpath``.""" + return re.sub(r"\[\d+\]", "", xpath) clean_xpath = remove_all_indices(abbreviated_xpath) return self._survey_elements.get(clean_xpath) def get_default_language(self): - if not hasattr(self, '_default_language'): - self._default_language = \ - self.survey.to_json_dict().get('default_language') + """Returns the default language""" + if not hasattr(self, "_default_language"): + self._default_language = self.survey.to_json_dict().get("default_language") return self._default_language default_language = property(get_default_language) def get_language(self, languages, language_index=0): + """Returns the language at the given index.""" language = None - if isinstance(languages, list) and len(languages): + if isinstance(languages, list) and languages: if self.default_language in languages: language_index = languages.index(self.default_language) @@ -537,6 +624,7 @@ def get_language(self, languages, language_index=0): return language def get_label(self, abbreviated_xpath, elem=None, language=None): + """Returns the label of given xpath.""" elem = self.get_element(abbreviated_xpath) if elem is None else elem if elem: @@ -547,28 +635,29 @@ def get_label(self, abbreviated_xpath, elem=None, language=None): label = label[language] else: language = self.get_language(list(label)) - label = label[language] if language else '' + label = label[language] if language else "" return label + return None def get_xpath_cmp(self): + """Compare two xpaths""" if not hasattr(self, "_xpaths"): - self._xpaths = [ - e.get_abbreviated_xpath() for e in self.survey_elements - ] + self._xpaths = [e.get_abbreviated_xpath() for e in self.survey_elements] + # pylint: disable=invalid-name def xpath_cmp(x, y): # For the moment, we aren't going to worry about repeating # nodes. - new_x = re.sub(r"\[\d+\]", u"", x) - new_y = re.sub(r"\[\d+\]", u"", y) + new_x = re.sub(r"\[\d+\]", "", x) + new_y = re.sub(r"\[\d+\]", "", y) if new_x == new_y: return cmp(x, y) if new_x not in self._xpaths and new_y not in self._xpaths: return 0 - elif new_x not in self._xpaths: + if new_x not in self._xpaths: return 1 - elif new_y not in self._xpaths: + if new_y not in self._xpaths: return -1 return cmp(self._xpaths.index(new_x), self._xpaths.index(new_y)) @@ -591,6 +680,7 @@ def get_variable_name(self, abbreviated_xpath): header = self._headers[i] if not hasattr(self, "_variable_names"): + # pylint: disable=import-outside-toplevel from onadata.apps.viewer.models.column_rename import ColumnRename self._variable_names = ColumnRename.get_dict() @@ -601,32 +691,36 @@ def get_variable_name(self, abbreviated_xpath): return header def get_list_of_parsed_instances(self, flat=True): + """Return an iterator of all parsed instances.""" for i in queryset_iterator(self.instances_for_export(self)): - # TODO: there is information we want to add in parsed xforms. yield i.get_dict(flat=flat) - def _rename_key(self, d, old_key, new_key): - if new_key in d: - raise AssertionError(d) - d[new_key] = d[old_key] - del d[old_key] - - def _expand_geocodes(self, d, key, e): - if e and e.bind.get(u"type") == u"geopoint": - geodata = d[key].split() - for i in range(len(geodata)): - new_key = "%s_%s" % (key, self.geodata_suffixes[i]) - d[new_key] = geodata[i] + def _rename_key(self, item, old_key, new_key): + """Moves a value in item at old_key to new_key""" + if new_key in item: + raise AssertionError(item) + item[new_key] = item[old_key] + del item[old_key] + + def _expand_geocodes(self, item, key, elem): + """Expands a geopoint into latitude, longitude, altitude, precision.""" + if elem and elem.bind.get("type") == "geopoint": + geodata = item[key].split() + for i, v in enumerate(geodata): + new_key = f"{key}_{self.GEODATA_SUFFIXES[i]}" + item[new_key] = v def get_data_for_excel(self): - for d in self.get_list_of_parsed_instances(): - for key in list(d): - e = self.get_element(key) - _expand_select_all_that_apply(d, key, e) - self._expand_geocodes(d, key, e) - yield d - - def _mark_start_time_boolean(self): + """Returns submissions with select all and geopoint fields expanded""" + for row in self.get_list_of_parsed_instances(): + for key in list(row): + elem = self.get_element(key) + _expand_select_all_that_apply(row, key, elem) + self._expand_geocodes(row, key, elem) + yield row + + def mark_start_time_boolean(self): + """Sets True the `self.has_start_time` if the form has a start meta question.""" starttime_substring = 'jr:preloadParams="start"' if self.xml.find(starttime_substring) != -1: self.has_start_time = True @@ -634,15 +728,14 @@ def _mark_start_time_boolean(self): self.has_start_time = False def get_survey_elements_of_type(self, element_type): - return [ - e for e in self.get_survey_elements() if e.type == element_type - ] + """Returns all survey elements of type ``element_type``.""" + return [e for e in self.get_survey_elements() if e.type == element_type] + # pylint: disable=invalid-name def get_survey_elements_with_choices(self): - if not hasattr(self, '_survey_elements_with_choices'): - choices_type = [ - constants.SELECT_ONE, constants.SELECT_ALL_THAT_APPLY - ] + """Returns all survey elements of type SELECT_ONE and SELECT_ALL_THAT_APPLY.""" + if not hasattr(self, "_survey_elements_with_choices"): + choices_type = [constants.SELECT_ONE, constants.SELECT_ALL_THAT_APPLY] self._survey_elements_with_choices = [ e for e in self.get_survey_elements() if e.type in choices_type @@ -654,11 +747,17 @@ def get_select_one_xpaths(self): """ Returns abbreviated_xpath for SELECT_ONE questions in the survey. """ - if not hasattr(self, '_select_one_xpaths'): + if not hasattr(self, "_select_one_xpaths"): self._select_one_xpaths = [ - e.get_abbreviated_xpath() for e in sum([ - self.get_survey_elements_of_type(select) - for select in [constants.SELECT_ONE]], [])] + e.get_abbreviated_xpath() + for e in sum( + [ + self.get_survey_elements_of_type(select) + for select in [constants.SELECT_ONE] + ], + [], + ) + ] return self._select_one_xpaths @@ -667,20 +766,27 @@ def get_select_multiple_xpaths(self): Returns abbreviated_xpath for SELECT_ALL_THAT_APPLY questions in the survey. """ - if not hasattr(self, '_select_multiple_xpaths'): + if not hasattr(self, "_select_multiple_xpaths"): self._select_multiple_xpaths = [ - e.get_abbreviated_xpath() for e in sum([ - self.get_survey_elements_of_type(select) - for select in [constants.SELECT_ALL_THAT_APPLY]], [])] + e.get_abbreviated_xpath() + for e in sum( + [ + self.get_survey_elements_of_type(select) + for select in [constants.SELECT_ALL_THAT_APPLY] + ], + [], + ) + ] return self._select_multiple_xpaths def get_media_survey_xpaths(self): + """Returns all survey element abbreviated_xpath of type in KNOWN_MEDIA_TYPES""" return [ e.get_abbreviated_xpath() - for e in sum([ - self.get_survey_elements_of_type(m) for m in KNOWN_MEDIA_TYPES - ], []) + for e in sum( + [self.get_survey_elements_of_type(m) for m in KNOWN_MEDIA_TYPES], [] + ) ] def get_osm_survey_xpaths(self): @@ -689,108 +795,121 @@ def get_osm_survey_xpaths(self): """ return [ elem.get_abbreviated_xpath() - for elem in self.get_survey_elements_of_type('osm')] + for elem in self.get_survey_elements_of_type("osm") + ] -@python_2_unicode_compatible +# pylint: disable=too-many-instance-attributes class XForm(XFormMixin, BaseModel): - CLONED_SUFFIX = '_cloned' + """XForm model - stores the XLSForm and related data.""" + + CLONED_SUFFIX = "_cloned" MAX_ID_LENGTH = 100 xls = models.FileField(upload_to=upload_to, null=True) - json = models.TextField(default=u'') - description = models.TextField(default=u'', null=True, blank=True) + # pylint: disable=no-member + json = models.JSONField(default=dict) + description = models.TextField(default="", null=True, blank=True) xml = models.TextField() user = models.ForeignKey( - User, related_name='xforms', null=True, on_delete=models.CASCADE) + User, related_name="xforms", null=True, on_delete=models.CASCADE + ) require_auth = models.BooleanField(default=False) shared = models.BooleanField(default=False) shared_data = models.BooleanField(default=False) downloadable = models.BooleanField(default=True) allows_sms = models.BooleanField(default=False) encrypted = models.BooleanField(default=False) - deleted_by = models.ForeignKey(User, related_name='xform_deleted_by', - null=True, on_delete=models.SET_NULL, - default=None, blank=True) + deleted_by = models.ForeignKey( + User, + related_name="xform_deleted_by", + null=True, + on_delete=models.SET_NULL, + default=None, + blank=True, + ) # the following fields are filled in automatically sms_id_string = models.SlugField( editable=False, - verbose_name=ugettext_lazy("SMS ID"), + verbose_name=gettext_lazy("SMS ID"), max_length=MAX_ID_LENGTH, - default='') + default="", + ) id_string = models.SlugField( - editable=False, - verbose_name=ugettext_lazy("ID"), - max_length=MAX_ID_LENGTH) + editable=False, verbose_name=gettext_lazy("ID"), max_length=MAX_ID_LENGTH + ) title = models.CharField(editable=False, max_length=XFORM_TITLE_LENGTH) date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) deleted_at = models.DateTimeField(blank=True, null=True) last_submission_time = models.DateTimeField(blank=True, null=True) has_start_time = models.BooleanField(default=False) - uuid = models.CharField(max_length=36, default=u'', db_index=True) - public_key = models.TextField(default='', blank=True, null=True) + uuid = models.CharField(max_length=36, default="", db_index=True) + public_key = models.TextField(default="", blank=True, null=True) - uuid_regex = re.compile(r'(.*?id="[^"]+">)(.*)(.*)', - re.DOTALL) - instance_id_regex = re.compile(r'.*?id="([^"]+)".*', - re.DOTALL) + uuid_regex = re.compile(r'(.*?id="[^"]+">)(.*)(.*)', re.DOTALL) + instance_id_regex = re.compile(r'.*?id="([^"]+)".*', re.DOTALL) uuid_node_location = 2 uuid_bind_location = 4 - bamboo_dataset = models.CharField(max_length=60, default=u'') + bamboo_dataset = models.CharField(max_length=60, default="") instances_with_geopoints = models.BooleanField(default=False) instances_with_osm = models.BooleanField(default=False) num_of_submissions = models.IntegerField(default=0) - version = models.CharField( - max_length=XFORM_TITLE_LENGTH, null=True, blank=True) - project = models.ForeignKey('Project', on_delete=models.CASCADE) + version = models.CharField(max_length=XFORM_TITLE_LENGTH, null=True, blank=True) + project = models.ForeignKey("Project", on_delete=models.CASCADE) created_by = models.ForeignKey( - User, null=True, blank=True, on_delete=models.SET_NULL) + User, null=True, blank=True, on_delete=models.SET_NULL + ) metadata_set = GenericRelation( - 'main.MetaData', - content_type_field='content_type_id', - object_id_field="object_id") + "main.MetaData", + content_type_field="content_type", + object_id_field="object_id", + ) has_hxl_support = models.BooleanField(default=False) last_updated_at = models.DateTimeField(auto_now=True) - hash = models.CharField(_("Hash"), max_length=36, blank=True, null=True, - default=None) + hash = models.CharField( + _("Hash"), max_length=36, blank=True, null=True, default=None + ) # XForm was created as a merged dataset is_merged_dataset = models.BooleanField(default=False) tags = TaggableManager() class Meta: - app_label = 'logger' - unique_together = (("user", "id_string", "project"), - ("user", "sms_id_string", "project")) - verbose_name = ugettext_lazy("XForm") - verbose_name_plural = ugettext_lazy("XForms") - ordering = ("pk", ) + app_label = "logger" + unique_together = ( + ("user", "id_string", "project"), + ("user", "sms_id_string", "project"), + ) + verbose_name = gettext_lazy("XForm") + verbose_name_plural = gettext_lazy("XForms") + ordering = ("pk",) permissions = ( ("view_xform_all", _("Can view all associated data")), ("view_xform_data", _("Can view submitted data")), ("report_xform", _("Can make submissions to the form")), - ("move_xform", _(u"Can move form between projects")), - ("transfer_xform", _(u"Can transfer form ownership.")), - ("can_export_xform_data", _(u"Can export form data")), - ("delete_submission", _(u"Can delete submissions from form")), + ("move_xform", _("Can move form between projects")), + ("transfer_xform", _("Can transfer form ownership.")), + ("can_export_xform_data", _("Can export form data")), + ("delete_submission", _("Can delete submissions from form")), ) def file_name(self): + """Returns the XML filename based on the ``self.id_string``.""" return self.id_string + ".xml" def url(self): + """Returns the download URL for the XForm.""" return reverse( "download_xform", - kwargs={ - "username": self.user.username, - "id_string": self.id_string - }) + kwargs={"username": self.user.username, "id_string": self.id_string}, + ) @property def has_instances_with_geopoints(self): + """Returns instances with geopoints.""" return self.instances_with_geopoints def _set_id_string(self): @@ -801,7 +920,7 @@ def _set_id_string(self): def _set_title(self): xml = re.sub(r"\s+", " ", self.xml) - matches = title_pattern.findall(xml) + matches = TITLE_PATTERN.findall(xml) if len(matches) != 1: raise XLSFormError(_("There should be a single title."), matches) @@ -809,118 +928,143 @@ def _set_title(self): if matches: title_xml = matches[0][:XFORM_TITLE_LENGTH] else: - title_xml = self.title[:XFORM_TITLE_LENGTH] if self.title else '' + title_xml = self.title[:XFORM_TITLE_LENGTH] if self.title else "" if self.title and title_xml != self.title: title_xml = self.title[:XFORM_TITLE_LENGTH] - if isinstance(self.xml, b): - self.xml = self.xml.decode('utf-8') - self.xml = title_pattern.sub(u"%s" % title_xml, - self.xml) - self._set_hash() + if isinstance(self.xml, bytes): + self.xml = self.xml.decode("utf-8") + self.xml = TITLE_PATTERN.sub(f"{title_xml}", self.xml) + self.set_hash() if contains_xml_invalid_char(title_xml): raise XLSFormError( - _("Title shouldn't have any invalid xml " - "characters ('>' '&' '<')")) + _("Title shouldn't have any invalid xml " "characters ('>' '&' '<')") + ) # Capture urls within form title - if re.search(r"^(?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~:/?#[\]@!\$&'\(\)\*\+,;=.]+$", self.title): # noqa - raise XLSFormError( - _("Invalid title value; value shouldn't match a URL")) + if re.search( + r"^(?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~:/?#[\]@!\$&'\(\)\*\+,;=.]+$", + self.title, + ): # noqa + raise XLSFormError(_("Invalid title value; value shouldn't match a URL")) self.title = title_xml - def _set_hash(self): + def get_hash(self): + """Returns the MD5 hash of the forms XML content prefixed by 'md5:'""" + md5_hash = hashlib.new( + "md5", self.xml.encode("utf-8"), usedforsecurity=False + ).hexdigest() + return f"md5:{md5_hash}" + + def set_hash(self): + """Sets the MD5 hash of the form.""" self.hash = self.get_hash() def _set_encrypted_field(self): - if self.json and self.json != '': - json_dict = json.loads(self.json) - self.encrypted = 'public_key' in json_dict + if self.json and self.json != "": + json_dict = self.json_dict() + self.encrypted = "public_key" in json_dict def _set_public_key_field(self): - if self.json and self.json != '': + if self.json and self.json != "": if self.num_of_submissions == 0 and self.public_key: - json_dict = json.loads(self.json) - json_dict['public_key'] = self.public_key + json_dict = self.json_dict() + json_dict["public_key"] = self.public_key survey = create_survey_element_from_dict(json_dict) - self.json = survey.to_json() + self.json = survey.to_json_dict() self.xml = survey.to_xml() self._set_encrypted_field() + def json_dict(self): + """Returns the `self.json` field data as a dict.""" + if isinstance(self.json, dict): + return self.json + + return json.loads(self.json) + def update(self, *args, **kwargs): - super(XForm, self).save(*args, **kwargs) + """Persists the form to the DB.""" + super().save(*args, **kwargs) - def save(self, *args, **kwargs): - update_fields = kwargs.get('update_fields') + # pylint: disable=too-many-branches,arguments-differ + def save(self, *args, **kwargs): # noqa: MC0001 + """Sets additional form properties before saving to the DB""" + update_fields = kwargs.get("update_fields") if update_fields: - kwargs['update_fields'] = list( - set(list(update_fields) + ['date_modified'])) - if update_fields is None or 'title' in update_fields: + kwargs["update_fields"] = list(set(list(update_fields) + ["date_modified"])) + if update_fields is None or "title" in update_fields: self._set_title() if self.pk is None: - self._set_hash() - if update_fields is None or 'encrypted' in update_fields: + self.set_hash() + if update_fields is None or "encrypted" in update_fields: self._set_encrypted_field() - if update_fields is None or 'id_string' in update_fields: + if update_fields is None or "id_string" in update_fields: old_id_string = self.id_string if not self.deleted_at: self._set_id_string() # check if we have an existing id_string, # if so, the one must match but only if xform is NOT new - if self.pk and old_id_string and old_id_string != self.id_string \ - and self.num_of_submissions > 0: + if ( + self.pk + and old_id_string + and old_id_string != self.id_string + and self.num_of_submissions > 0 + ): raise XLSFormError( - _(u"Your updated form's id_string '%(new_id)s' must match " - "the existing forms' id_string '%(old_id)s'." % - {'new_id': self.id_string, - 'old_id': old_id_string})) - - if getattr(settings, 'STRICT', True) and \ - not re.search(r"^[\w-]+$", self.id_string): + _( + "Your updated form's id_string '%(new_id)s' must match " + "the existing forms' id_string '%(old_id)s'." + % {"new_id": self.id_string, "old_id": old_id_string} + ) + ) + + if getattr(settings, "STRICT", True) and not re.search( + r"^[\w-]+$", self.id_string + ): raise XLSFormError( - _(u'In strict mode, the XForm ID must be a ' - 'valid slug and contain no spaces. Please ensure' - ' that you have set an id_string in the settings sheet ' - 'or have modified the filename to not contain' - ' any spaces.')) - - if not self.sms_id_string and (update_fields is None or - 'id_string' in update_fields): - try: - # try to guess the form's wanted sms_id_string - # from it's json rep (from XLSForm) - # otherwise, use id_string to ensure uniqueness - self.sms_id_string = json.loads(self.json).get( - 'sms_keyword', self.id_string) - except Exception: - self.sms_id_string = self.id_string - - if update_fields is None or 'public_key' in update_fields: + _( + "In strict mode, the XForm ID must be a " + "valid slug and contain no spaces. Please ensure" + " that you have set an id_string in the settings sheet " + "or have modified the filename to not contain" + " any spaces." + ) + ) + + if not self.sms_id_string and ( + update_fields is None or "id_string" in update_fields + ): + json_dict = self.json_dict() + self.sms_id_string = json_dict.get("sms_keyword", self.id_string) + + if update_fields is None or "public_key" in update_fields: self._set_public_key_field() - if 'skip_xls_read' in kwargs: - del kwargs['skip_xls_read'] + if "skip_xls_read" in kwargs: + del kwargs["skip_xls_read"] - if (self.id_string and len( - self.id_string) > self.MAX_ID_LENGTH) or \ - (self.sms_id_string and len( - self.sms_id_string) > self.MAX_ID_LENGTH): + if (self.id_string and len(self.id_string) > self.MAX_ID_LENGTH) or ( + self.sms_id_string and len(self.sms_id_string) > self.MAX_ID_LENGTH + ): raise XLSFormError( - _(u'The XForm id_string provided exceeds %s characters.' - ' Please change the "id_string" or "form_id" values' - 'in settings sheet or reduce the file name if you do' - ' not have a settings sheets.' % self.MAX_ID_LENGTH)) + _( + f"The XForm id_string provided exceeds {self.MAX_ID_LENGTH} characters." + f' Please change the "id_string" or "form_id" values' + f"in settings sheet or reduce the file name if you do" + f" not have a settings sheets." + ) + ) is_version_available = self.version is not None if is_version_available and contains_xml_invalid_char(self.version): raise XLSFormError( - _("Version shouldn't have any invalid " - "characters ('>' '&' '<')")) + _("Version shouldn't have any invalid " "characters ('>' '&' '<')") + ) self.description = conditional_escape(self.description) - super(XForm, self).save(*args, **kwargs) + super().save(*args, **kwargs) def __str__(self): return getattr(self, "id_string", "") @@ -934,80 +1078,95 @@ def soft_delete(self, user=None): without violating the uniqueness constraint. Also soft deletes associated dataviews """ - soft_deletion_time = timezone.now() - deletion_suffix = soft_deletion_time.strftime('-deleted-at-%s') + deletion_suffix = soft_deletion_time.strftime("-deleted-at-%s") self.deleted_at = soft_deletion_time self.id_string += deletion_suffix self.sms_id_string += deletion_suffix self.downloadable = False # only take the first 100 characters (within the set max_length) - self.id_string = self.id_string[:self.MAX_ID_LENGTH] - self.sms_id_string = self.sms_id_string[:self.MAX_ID_LENGTH] - - update_fields = ['date_modified', 'deleted_at', 'id_string', - 'sms_id_string', 'downloadable'] + self.id_string = self.id_string[: self.MAX_ID_LENGTH] + self.sms_id_string = self.sms_id_string[: self.MAX_ID_LENGTH] + + update_fields = [ + "date_modified", + "deleted_at", + "id_string", + "sms_id_string", + "downloadable", + ] if user is not None: self.deleted_by = user - update_fields.append('deleted_by') + update_fields.append("deleted_by") self.save(update_fields=update_fields) # Delete associated filtered datasets for dataview in self.dataview_set.all(): dataview.soft_delete(user) # Delete associated Merged-Datasets - for merged_dataset in self.mergedxform_ptr.filter( - deleted_at__isnull=True): + for merged_dataset in self.mergedxform_ptr.filter(deleted_at__isnull=True): merged_dataset.soft_delete(user) # Delete associated Form Media Files - for metadata in self.metadata_set.filter( - deleted_at__isnull=True): + for metadata in self.metadata_set.filter(deleted_at__isnull=True): metadata.soft_delete() + clear_project_cache(self.project_id) def submission_count(self, force_update=False): + """Returns the form's number of submission.""" if self.num_of_submissions == 0 or force_update: if self.is_merged_dataset: - count = self.mergedxform.xforms.aggregate( - num=Sum('num_of_submissions')).get('num') or 0 + count = ( + self.mergedxform.xforms.aggregate( + num=Sum("num_of_submissions") + ).get("num") + or 0 + ) else: count = self.instances.filter(deleted_at__isnull=True).count() if count != self.num_of_submissions: self.num_of_submissions = count - self.save(update_fields=['num_of_submissions']) + self.save(update_fields=["num_of_submissions"]) # clear cache - key = '{}{}'.format(XFORM_COUNT, self.pk) + key = f"{XFORM_COUNT}{self.pk}" safe_delete(key) return self.num_of_submissions - submission_count.short_description = ugettext_lazy("Submission Count") + submission_count.short_description = gettext_lazy("Submission Count") @property def submission_count_for_today(self): + """Returns the submissions count for the current day.""" current_timzone_name = timezone.get_current_timezone_name() current_timezone = pytz.timezone(current_timzone_name) today = datetime.today() current_date = current_timezone.localize( - datetime(today.year, today.month, today.day)).isoformat() - count = cache.get( - f"{XFORM_SUBMISSION_COUNT_FOR_DAY}{self.id}") if cache.get( - f"{XFORM_SUBMISSION_COUNT_FOR_DAY_DATE}{self.id}" - ) == current_date else 0 + datetime(today.year, today.month, today.day) + ).isoformat() + count = ( + cache.get(f"{XFORM_SUBMISSION_COUNT_FOR_DAY}{self.id}") + if cache.get(f"{XFORM_SUBMISSION_COUNT_FOR_DAY_DATE}{self.id}") + == current_date + else 0 + ) return count def geocoded_submission_count(self): """Number of geocoded submissions.""" return self.instances.filter( - deleted_at__isnull=True, geom__isnull=False).count() + deleted_at__isnull=True, geom__isnull=False + ).count() def time_of_last_submission(self): + """Returns the timestamp of when the latest submission was created.""" if self.last_submission_time is None and self.num_of_submissions > 0: try: - last_submission = self.instances.\ - filter(deleted_at__isnull=True).latest("date_created") + last_submission = self.instances.filter(deleted_at__isnull=True).latest( + "date_created" + ) except ObjectDoesNotExist: pass else: @@ -1016,100 +1175,88 @@ def time_of_last_submission(self): return self.last_submission_time def time_of_last_submission_update(self): + """Returns the timestamp of the last updated submission for the form.""" + last_submission_time = None try: # we also consider deleted instances in this case - return self.instances.latest("date_modified").date_modified + last_submission_time = self.instances.latest("date_modified").date_modified except ObjectDoesNotExist: pass - def get_hash(self): - return u'md5:%s' % md5(self.xml.encode('utf8')).hexdigest() + return last_submission_time @property def can_be_replaced(self): + """Returns True if the form has zero submissions - forms with zero permissions + can be replaced.""" return self.num_of_submissions == 0 @classmethod def public_forms(cls): + """Returns a queryset of public forms i.e. shared = True""" return cls.objects.filter(shared=True) +# pylint: disable=unused-argument def update_profile_num_submissions(sender, instance, **kwargs): + """Reduce the user's number of submissions on deletions.""" profile_qs = User.profile.get_queryset() try: - profile = profile_qs.select_for_update()\ - .get(pk=instance.user.profile.pk) + profile = profile_qs.select_for_update().get(pk=instance.user.profile.pk) except ObjectDoesNotExist: pass else: profile.num_of_submissions -= instance.num_of_submissions - if profile.num_of_submissions < 0: - profile.num_of_submissions = 0 + profile.num_of_submissions = max(profile.num_of_submissions, 0) profile.save() post_delete.connect( update_profile_num_submissions, sender=XForm, - dispatch_uid='update_profile_num_submissions') + dispatch_uid="update_profile_num_submissions", +) def clear_project_cache(project_id): - safe_delete('{}{}'.format(PROJ_OWNER_CACHE, project_id)) - safe_delete('{}{}'.format(PROJ_FORMS_CACHE, project_id)) - safe_delete('{}{}'.format(PROJ_BASE_FORMS_CACHE, project_id)) - safe_delete('{}{}'.format(PROJ_SUB_DATE_CACHE, project_id)) - safe_delete('{}{}'.format(PROJ_NUM_DATASET_CACHE, project_id)) - - -def set_object_permissions(sender, instance=None, created=False, **kwargs): - # clear cache - project = instance.project - project.refresh_from_db() - clear_project_cache(project.pk) - safe_delete('{}{}'.format(IS_ORG, instance.pk)) - - if created: - from onadata.libs.permissions import OwnerRole - OwnerRole.add(instance.user, instance) - - if instance.created_by and instance.user != instance.created_by: - OwnerRole.add(instance.created_by, instance) - - from onadata.libs.utils.project_utils import set_project_perms_to_xform - set_project_perms_to_xform(instance, project) - - -post_save.connect( - set_object_permissions, - sender=XForm, - dispatch_uid='xform_object_permissions') + """Clear project cache""" + safe_delete(f"{PROJ_OWNER_CACHE}{project_id}") + safe_delete(f"{PROJ_FORMS_CACHE}{project_id}") + safe_delete(f"{PROJ_BASE_FORMS_CACHE}{project_id}") + safe_delete(f"{PROJ_SUB_DATE_CACHE}{project_id}") + safe_delete(f"{PROJ_NUM_DATASET_CACHE}{project_id}") +# pylint: disable=unused-argument def save_project(sender, instance=None, created=False, **kwargs): - instance.project.save(update_fields=['date_modified']) + """Update the date_modified field in the XForm's project.""" + clear_project_cache(instance.project_id) + instance.project.save(update_fields=["date_modified"]) -pre_save.connect(save_project, sender=XForm, dispatch_uid='save_project_xform') +pre_save.connect(save_project, sender=XForm, dispatch_uid="save_project_xform") +# pylint: disable=unused-argument def xform_post_delete_callback(sender, instance, **kwargs): + """Clear project cache after deleting an XForm.""" if instance.project_id: clear_project_cache(instance.project_id) post_delete.connect( - xform_post_delete_callback, - sender=XForm, - dispatch_uid='xform_post_delete_callback') + xform_post_delete_callback, sender=XForm, dispatch_uid="xform_post_delete_callback" +) +# pylint: disable=too-few-public-methods class XFormUserObjectPermission(UserObjectPermissionBase): """Guardian model to create direct foreign keys.""" content_object = models.ForeignKey(XForm, on_delete=models.CASCADE) +# pylint: disable=too-few-public-methods class XFormGroupObjectPermission(GroupObjectPermissionBase): """Guardian model to create direct foreign keys.""" @@ -1121,12 +1268,10 @@ def check_xform_uuid(new_uuid): Checks if a new_uuid has already been used, if it has it raises the exception DuplicateUUIDError. """ - count = XForm.objects.filter(uuid=new_uuid, - deleted_at__isnull=True).count() + count = XForm.objects.filter(uuid=new_uuid, deleted_at__isnull=True).count() if count > 0: - raise DuplicateUUIDError( - "An xform with uuid: %s already exists" % new_uuid) + raise DuplicateUUIDError(f"An xform with uuid: {new_uuid} already exists") def update_xform_uuid(username, id_string, new_uuid): diff --git a/onadata/apps/logger/models/xform_version.py b/onadata/apps/logger/models/xform_version.py index 3a6cc2c012..277a9bd0c7 100644 --- a/onadata/apps/logger/models/xform_version.py +++ b/onadata/apps/logger/models/xform_version.py @@ -13,15 +13,15 @@ class XFormVersion(models.Model): storage backend for utilization in the future when a user requires the previous XForm versions XML or JSON. """ + xform = models.ForeignKey( - 'logger.XForm', on_delete=models.CASCADE, related_name='versions') + "logger.XForm", on_delete=models.CASCADE, related_name="versions" + ) xls = models.FileField() version = models.CharField(max_length=100) date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) - created_by = models.ForeignKey( - 'auth.User', on_delete=models.SET_NULL, null=True - ) + created_by = models.ForeignKey("auth.User", on_delete=models.SET_NULL, null=True) xml = models.TextField() json = models.TextField() @@ -29,4 +29,4 @@ def __str__(self): return f"{self.xform.title}-{self.version}" class Meta: - unique_together = ['xform', 'version'] + unique_together = ["xform", "version"] diff --git a/onadata/apps/logger/signals.py b/onadata/apps/logger/signals.py index 9e044efdab..316c0a998d 100644 --- a/onadata/apps/logger/signals.py +++ b/onadata/apps/logger/signals.py @@ -1,16 +1,27 @@ +# -*- coding: utf-8 -*- +""" +logger signals module +""" from django.db.models.signals import post_save from django.dispatch import receiver from onadata.apps.logger.models import MergedXForm +from onadata.apps.logger.models import XForm +from onadata.apps.logger.models.xform import clear_project_cache from onadata.libs.permissions import OwnerRole +from onadata.libs.utils.cache_tools import ( + IS_ORG, + safe_delete, +) from onadata.libs.utils.project_utils import set_project_perms_to_xform +# pylint: disable=unused-argument @receiver( - post_save, - sender=MergedXForm, - dispatch_uid='set_project_perms_to_merged_xform') -def set_object_permissions(sender, instance=None, created=False, **kwargs): + post_save, sender=MergedXForm, dispatch_uid="set_project_perms_to_merged_xform" +) +def set_project_object_permissions(sender, instance=None, created=False, **kwargs): + """Apply project permission to the merged form.""" if created: OwnerRole.add(instance.user, instance) OwnerRole.add(instance.user, instance.xform_ptr) @@ -21,3 +32,26 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): set_project_perms_to_xform(instance, instance.project) set_project_perms_to_xform(instance.xform_ptr, instance.project) + + +# pylint: disable=unused-argument +def set_xform_object_permissions(sender, instance=None, created=False, **kwargs): + """Apply project permissions to the user that created the form.""" + # clear cache + project = instance.project + project.refresh_from_db() + clear_project_cache(project.pk) + safe_delete(f"{IS_ORG}{instance.pk}") + + if created: + OwnerRole.add(instance.user, instance) + + if instance.created_by and instance.user != instance.created_by: + OwnerRole.add(instance.created_by, instance) + + set_project_perms_to_xform(instance, project) + + +post_save.connect( + set_xform_object_permissions, sender=XForm, dispatch_uid="xform_object_permissions" +) diff --git a/onadata/apps/logger/tests/management/commands/test_recover_deleted_attachments.py b/onadata/apps/logger/tests/management/commands/test_recover_deleted_attachments.py index 03ba6bb992..6bf2859be2 100644 --- a/onadata/apps/logger/tests/management/commands/test_recover_deleted_attachments.py +++ b/onadata/apps/logger/tests/management/commands/test_recover_deleted_attachments.py @@ -9,13 +9,15 @@ from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.import_tools import django_file -from onadata.apps.logger.management.commands.recover_deleted_attachments \ - import recover_deleted_attachments +from onadata.apps.logger.management.commands.recover_deleted_attachments import ( + recover_deleted_attachments, +) from onadata.libs.utils.logger_tools import create_instance class TestRecoverDeletedAttachments(TestBase): """TestRecoverDeletedAttachments Class""" + # pylint: disable=invalid-name def test_recovers_wrongly_deleted_attachments(self): """ @@ -40,20 +42,28 @@ def test_recovers_wrongly_deleted_attachments(self): 1300221157303.jpg """ - media_root = (f'{settings.PROJECT_ROOT}/apps/logger/tests/Health' - '_2011_03_13.xml_2011-03-15_20-30-28/') + media_root = ( + f"{settings.PROJECT_ROOT}/apps/logger/tests/Health" + "_2011_03_13.xml_2011-03-15_20-30-28/" + ) image_media = django_file( - path=f'{media_root}1300221157303.jpg', field_name='image', - content_type='image/jpeg') + path=f"{media_root}1300221157303.jpg", + field_name="image", + content_type="image/jpeg", + ) file_media = django_file( - path=f'{media_root}Health_2011_03_13.xml_2011-03-15_20-30-28.xml', - field_name='file', content_type='text/xml') + path=f"{media_root}Health_2011_03_13.xml_2011-03-15_20-30-28.xml", + field_name="file", + content_type="text/xml", + ) instance = create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[file_media, image_media]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[file_media, image_media], + ) self.assertEqual( - instance.attachments.filter(deleted_at__isnull=True).count(), 2) + instance.attachments.filter(deleted_at__isnull=True).count(), 2 + ) attachment = instance.attachments.first() # Soft delete attachment @@ -62,13 +72,15 @@ def test_recovers_wrongly_deleted_attachments(self): attachment.save() self.assertEqual( - instance.attachments.filter(deleted_at__isnull=True).count(), 1) + instance.attachments.filter(deleted_at__isnull=True).count(), 1 + ) # Attempt recovery of attachment recover_deleted_attachments(form_id=instance.xform.id) self.assertEqual( - instance.attachments.filter(deleted_at__isnull=True).count(), 2) + instance.attachments.filter(deleted_at__isnull=True).count(), 2 + ) attachment.refresh_from_db() self.assertIsNone(attachment.deleted_at) self.assertIsNone(attachment.deleted_by) diff --git a/onadata/apps/logger/tests/management/commands/test_remove_columns_from_briefcase_data.py b/onadata/apps/logger/tests/management/commands/test_remove_columns_from_briefcase_data.py index ea27e836af..a0ac779b00 100644 --- a/onadata/apps/logger/tests/management/commands/test_remove_columns_from_briefcase_data.py +++ b/onadata/apps/logger/tests/management/commands/test_remove_columns_from_briefcase_data.py @@ -5,8 +5,9 @@ from io import BytesIO from onadata.apps.main.tests.test_base import TestBase -from onadata.apps.logger.management.commands.\ - remove_columns_from_briefcase_data import remove_columns_from_xml +from onadata.apps.logger.management.commands.remove_columns_from_briefcase_data import ( + remove_columns_from_xml, +) from onadata.libs.utils.logger_tools import create_instance @@ -38,15 +39,16 @@ def test_removes_correct_columns(self): """ instance = create_instance( - self.user.username, BytesIO(xml_string.strip().encode('utf-8')), - media_files=[] + self.user.username, + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[], ) expected_xml = ( f'<{id_string} id="{id_string}">' "uuid:UJ5jz4EszdgH8uhy8nss1AsKaqBPO5VN7" "I love coding!" - f"") - modified_xml = remove_columns_from_xml( - instance.xml, ['first_name', 'photo']) + f"" + ) + modified_xml = remove_columns_from_xml(instance.xml, ["first_name", "photo"]) self.assertEqual(modified_xml, expected_xml) diff --git a/onadata/apps/logger/tests/management/commands/test_replace_form_id_root_node.py b/onadata/apps/logger/tests/management/commands/test_replace_form_id_root_node.py index d18cd1714d..f92036c396 100644 --- a/onadata/apps/logger/tests/management/commands/test_replace_form_id_root_node.py +++ b/onadata/apps/logger/tests/management/commands/test_replace_form_id_root_node.py @@ -8,8 +8,9 @@ from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.import_tools import django_file -from onadata.apps.logger.management.commands.replace_form_id_root_node \ - import replace_form_id_with_correct_root_node +from onadata.apps.logger.management.commands.replace_form_id_root_node import ( + replace_form_id_with_correct_root_node, +) from onadata.libs.utils.logger_tools import create_instance @@ -39,22 +40,30 @@ def test_replaces_form_id_root_node(self): 1300221157303.jpg """ - media_root = (f'{settings.PROJECT_ROOT}/apps/logger/tests/Health' - '_2011_03_13.xml_2011-03-15_20-30-28/') + media_root = ( + f"{settings.PROJECT_ROOT}/apps/logger/tests/Health" + "_2011_03_13.xml_2011-03-15_20-30-28/" + ) image_media = django_file( - path=f'{media_root}1300221157303.jpg', field_name='image', - content_type='image/jpeg') + path=f"{media_root}1300221157303.jpg", + field_name="image", + content_type="image/jpeg", + ) file_media = django_file( - path=f'{media_root}Health_2011_03_13.xml_2011-03-15_20-30-28.xml', - field_name='file', content_type='text/xml') + path=f"{media_root}Health_2011_03_13.xml_2011-03-15_20-30-28.xml", + field_name="file", + content_type="text/xml", + ) instance = create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[file_media, image_media]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[file_media, image_media], + ) # Attempt replacement of root node name replace_form_id_with_correct_root_node( - inst_id=instance.id, root='data', commit=True) + inst_id=instance.id, root="data", commit=True + ) instance.refresh_from_db() expected_xml = f""" diff --git a/onadata/apps/logger/tests/models/test_attachment.py b/onadata/apps/logger/tests/models/test_attachment.py index d6437ab78f..ddb7be623c 100644 --- a/onadata/apps/logger/tests/models/test_attachment.py +++ b/onadata/apps/logger/tests/models/test_attachment.py @@ -9,115 +9,105 @@ from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.models import Attachment, Instance -from onadata.apps.logger.models.attachment import (get_original_filename, - upload_to) +from onadata.apps.logger.models.attachment import get_original_filename, upload_to from onadata.libs.utils.image_tools import image_url class TestAttachment(TestBase): - def setUp(self): super(self.__class__, self).setUp() self._publish_transportation_form_and_submit_instance() self.media_file = "1335783522563.jpg" media_file = os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', self.surveys[0], self.media_file) + self.this_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + self.media_file, + ) self.instance = Instance.objects.all()[0] self.attachment = Attachment.objects.create( - instance=self.instance, - media_file=File(open(media_file, 'rb'), media_file)) + instance=self.instance, media_file=File(open(media_file, "rb"), media_file) + ) def test_mimetype(self): - self.assertEqual(self.attachment.mimetype, 'image/jpeg') + self.assertEqual(self.attachment.mimetype, "image/jpeg") def test_create_attachment_with_mimetype_more_than_50(self): media_file = os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', self.surveys[0], self.media_file) - media_file = File(open(media_file, 'rb'), media_file) + self.this_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + self.media_file, + ) + media_file = File(open(media_file, "rb"), media_file) with self.assertRaises(DataError): Attachment.objects.create( - instance=self.instance, - mimetype='a'*120, - media_file=media_file + instance=self.instance, mimetype="a" * 120, media_file=media_file ) pre_count = Attachment.objects.count() Attachment.objects.create( - instance=self.instance, - mimetype='a'*100, - media_file=media_file + instance=self.instance, mimetype="a" * 100, media_file=media_file ) self.assertEqual(pre_count + 1, Attachment.objects.count()) def test_create_attachment_with_media_file_length_more_the_100(self): with self.assertRaises(ValueError): - Attachment.objects.create( - instance=self.instance, - media_file='a'*300 - ) + Attachment.objects.create(instance=self.instance, media_file="a" * 300) pre_count = Attachment.objects.count() - Attachment.objects.create( - instance=self.instance, - media_file='a'*150 - ) + Attachment.objects.create(instance=self.instance, media_file="a" * 150) self.assertEqual(pre_count + 1, Attachment.objects.count()) def test_thumbnails(self): for attachment in Attachment.objects.filter(instance=self.instance): - url = image_url(attachment, 'small') - filename = attachment.media_file.name.replace('.jpg', '') - thumbnail = '%s-small.jpg' % filename - self.assertNotEqual( - url.find(thumbnail), -1) - for size in ['small', 'medium', 'large']: - thumbnail = '%s-%s.jpg' % (filename, size) - self.assertTrue( - default_storage.exists(thumbnail)) + url = image_url(attachment, "small") + filename = attachment.media_file.name.replace(".jpg", "") + thumbnail = "%s-small.jpg" % filename + self.assertNotEqual(url.find(thumbnail), -1) + for size in ["small", "medium", "large"]: + thumbnail = "%s-%s.jpg" % (filename, size) + self.assertTrue(default_storage.exists(thumbnail)) default_storage.delete(thumbnail) def test_create_thumbnails_command(self): call_command("create_image_thumbnails") for attachment in Attachment.objects.filter(instance=self.instance): - filename = attachment.media_file.name.replace('.jpg', '') - for size in ['small', 'medium', 'large']: - thumbnail = '%s-%s.jpg' % (filename, size) - self.assertTrue( - default_storage.exists(thumbnail)) + filename = attachment.media_file.name.replace(".jpg", "") + for size in ["small", "medium", "large"]: + thumbnail = "%s-%s.jpg" % (filename, size) + self.assertTrue(default_storage.exists(thumbnail)) check_datetime = timezone.now() # replace or regenerate thumbnails if they exist call_command("create_image_thumbnails", force=True) for attachment in Attachment.objects.filter(instance=self.instance): - filename = attachment.media_file.name.replace('.jpg', '') - for size in ['small', 'medium', 'large']: - thumbnail = '%s-%s.jpg' % (filename, size) - self.assertTrue( - default_storage.exists(thumbnail)) + filename = attachment.media_file.name.replace(".jpg", "") + for size in ["small", "medium", "large"]: + thumbnail = "%s-%s.jpg" % (filename, size) + self.assertTrue(default_storage.exists(thumbnail)) self.assertTrue( - default_storage.get_modified_time(thumbnail) > - check_datetime + default_storage.get_modified_time(thumbnail) > check_datetime ) default_storage.delete(thumbnail) def test_get_original_filename(self): self.assertEqual( - get_original_filename('submission.xml_K337n8u.enc'), - 'submission.xml.enc' + get_original_filename("submission.xml_K337n8u.enc"), "submission.xml.enc" ) self.assertEqual( - get_original_filename('submission.xml.enc'), - 'submission.xml.enc' + get_original_filename("submission.xml.enc"), "submission.xml.enc" ) self.assertEqual( - get_original_filename('submission_test.xml_K337n8u.enc'), - 'submission_test.xml.enc' + get_original_filename("submission_test.xml_K337n8u.enc"), + "submission_test.xml.enc", ) self.assertEqual( - get_original_filename('submission_random.enc'), - 'submission_random.enc' + get_original_filename("submission_random.enc"), "submission_random.enc" ) def test_upload_to(self): @@ -125,6 +115,9 @@ def test_upload_to(self): Test that upload to returns the correct path """ path = upload_to(self.attachment, self.attachment.filename) - self.assertEqual(path, - 'bob/attachments/{}_{}/1335783522563.jpg'.format( - self.xform.id, self.xform.id_string)) + self.assertEqual( + path, + "bob/attachments/{}_{}/1335783522563.jpg".format( + self.xform.id, self.xform.id_string + ), + ) diff --git a/onadata/apps/logger/tests/models/test_data_view.py b/onadata/apps/logger/tests/models/test_data_view.py index e63d25ac9f..573e0c4358 100644 --- a/onadata/apps/logger/tests/models/test_data_view.py +++ b/onadata/apps/logger/tests/models/test_data_view.py @@ -4,45 +4,20 @@ from django.db import connection from onadata.apps.main.tests.test_base import TestBase -from onadata.apps.api.tests.viewsets.test_abstract_viewset import\ - TestAbstractViewSet -from onadata.apps.logger.models.data_view import ( - append_where_list, - DataView) +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet +from onadata.apps.logger.models.data_view import append_where_list, DataView class TestDataView(TestBase): - def test_append_where_list(self): - json_str = 'json->>%s' - self.assertEqual( - append_where_list('<>', [], json_str), - [u'json->>%s <> %s'] - ) - self.assertEqual( - append_where_list('!=', [], json_str), - [u'json->>%s <> %s'] - ) - self.assertEqual( - append_where_list('=', [], json_str), - [u'json->>%s = %s'] - ) - self.assertEqual( - append_where_list('>', [], json_str), - [u'json->>%s > %s'] - ) - self.assertEqual( - append_where_list('<', [], json_str), - [u'json->>%s < %s'] - ) - self.assertEqual( - append_where_list('>=', [], json_str), - [u'json->>%s >= %s'] - ) - self.assertEqual( - append_where_list('<=', [], json_str), - [u'json->>%s <= %s'] - ) + json_str = "json->>%s" + self.assertEqual(append_where_list("<>", [], json_str), ["json->>%s <> %s"]) + self.assertEqual(append_where_list("!=", [], json_str), ["json->>%s <> %s"]) + self.assertEqual(append_where_list("=", [], json_str), ["json->>%s = %s"]) + self.assertEqual(append_where_list(">", [], json_str), ["json->>%s > %s"]) + self.assertEqual(append_where_list("<", [], json_str), ["json->>%s < %s"]) + self.assertEqual(append_where_list(">=", [], json_str), ["json->>%s >= %s"]) + self.assertEqual(append_where_list("<=", [], json_str), ["json->>%s <= %s"]) class TestIntegratedDataView(TestAbstractViewSet): @@ -62,26 +37,36 @@ def setUp(self): def _setup_dataview(self): xlsform_path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", "fixtures", - "tutorial.xlsx") + settings.PROJECT_ROOT, "libs", "tests", "utils", "fixtures", "tutorial.xlsx" + ) self._publish_xls_form_to_project(xlsform_path=xlsform_path) for x in range(1, 9): path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - 'tutorial', 'instances', 'uuid{}'.format(x), 'submission.xml') + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial", + "instances", + "uuid{}".format(x), + "submission.xml", + ) self._make_submission(path) x += 1 self._create_dataview() def test_generate_query_string_for_data_without_filter(self): - expected_sql = "SELECT json FROM "\ - "logger_instance WHERE xform_id = %s AND "\ - "CAST(json->>%s AS INT) > %s AND "\ - "CAST(json->>%s AS INT) < %s AND deleted_at IS NULL"\ - " ORDER BY id" + expected_sql = ( + "SELECT json FROM " + "logger_instance WHERE xform_id = %s AND " + "CAST(json->>%s AS INT) > %s AND " + "CAST(json->>%s AS INT) < %s AND deleted_at IS NULL" + " ORDER BY id" + ) (sql, columns, params) = DataView.generate_query_string( self.data_view, @@ -89,22 +74,25 @@ def test_generate_query_string_for_data_without_filter(self): self.limit, self.last_submission_time, self.all_data, - self.sort) + self.sort, + ) - self.assertEquals(sql, expected_sql) + self.assertEqual(sql, expected_sql) self.assertEqual(len(columns), 8) self.cursor.execute(sql, [str(i) for i in (params)]) results = self.cursor.fetchall() - self.assertEquals(len(results), 3) + self.assertEqual(len(results), 3) def test_generate_query_string_for_data_with_limit_filter(self): limit_filter = 1 - expected_sql = "SELECT json FROM logger_instance"\ - " WHERE xform_id = %s AND CAST(json->>%s AS INT) > %s"\ - " AND CAST(json->>%s AS INT) < %s AND deleted_at "\ - "IS NULL ORDER BY id LIMIT %s" + expected_sql = ( + "SELECT json FROM logger_instance" + " WHERE xform_id = %s AND CAST(json->>%s AS INT) > %s" + " AND CAST(json->>%s AS INT) < %s AND deleted_at " + "IS NULL ORDER BY id LIMIT %s" + ) (sql, columns, params) = DataView.generate_query_string( self.data_view, @@ -112,23 +100,26 @@ def test_generate_query_string_for_data_with_limit_filter(self): limit_filter, self.last_submission_time, self.all_data, - self.sort) + self.sort, + ) - self.assertEquals(sql, expected_sql) + self.assertEqual(sql, expected_sql) - records = [record for record in DataView.query_iterator(sql, - columns, - params, - self.count)] + records = [ + record + for record in DataView.query_iterator(sql, columns, params, self.count) + ] - self.assertEquals(len(records), limit_filter) + self.assertEqual(len(records), limit_filter) def test_generate_query_string_for_data_with_start_index_filter(self): start_index = 2 - expected_sql = "SELECT json FROM logger_instance WHERE"\ - " xform_id = %s AND CAST(json->>%s AS INT) > %s AND"\ - " CAST(json->>%s AS INT) < %s AND deleted_at IS NULL "\ - "ORDER BY id OFFSET %s" + expected_sql = ( + "SELECT json FROM logger_instance WHERE" + " xform_id = %s AND CAST(json->>%s AS INT) > %s AND" + " CAST(json->>%s AS INT) < %s AND deleted_at IS NULL " + "ORDER BY id OFFSET %s" + ) (sql, columns, params) = DataView.generate_query_string( self.data_view, @@ -136,26 +127,29 @@ def test_generate_query_string_for_data_with_start_index_filter(self): self.limit, self.last_submission_time, self.all_data, - self.sort) + self.sort, + ) - self.assertEquals(sql, expected_sql) + self.assertEqual(sql, expected_sql) - records = [record for record in DataView.query_iterator(sql, - columns, - params, - self.count)] - self.assertEquals(len(records), 1) - self.assertIn('name', records[0]) - self.assertIn('age', records[0]) - self.assertIn('gender', records[0]) - self.assertNotIn('pizza_type', records[0]) + records = [ + record + for record in DataView.query_iterator(sql, columns, params, self.count) + ] + self.assertEqual(len(records), 1) + self.assertIn("name", records[0]) + self.assertIn("age", records[0]) + self.assertIn("gender", records[0]) + self.assertNotIn("pizza_type", records[0]) def test_generate_query_string_for_data_with_sort_column_asc(self): sort = '{"age":1}' - expected_sql = "SELECT json FROM logger_instance WHERE"\ - " xform_id = %s AND CAST(json->>%s AS INT) > %s AND"\ - " CAST(json->>%s AS INT) < %s AND deleted_at IS NULL"\ - " ORDER BY json->>%s ASC" + expected_sql = ( + "SELECT json FROM logger_instance WHERE" + " xform_id = %s AND CAST(json->>%s AS INT) > %s AND" + " CAST(json->>%s AS INT) < %s AND deleted_at IS NULL" + " ORDER BY json->>%s ASC" + ) (sql, columns, params) = DataView.generate_query_string( self.data_view, @@ -163,23 +157,26 @@ def test_generate_query_string_for_data_with_sort_column_asc(self): self.limit, self.last_submission_time, self.all_data, - sort) + sort, + ) - self.assertEquals(sql, expected_sql) + self.assertEqual(sql, expected_sql) - records = [record for record in DataView.query_iterator(sql, - columns, - params, - self.count)] + records = [ + record + for record in DataView.query_iterator(sql, columns, params, self.count) + ] self.assertTrue(self.is_sorted_asc([r.get("age") for r in records])) def test_generate_query_string_for_data_with_sort_column_desc(self): sort = '{"age": -1}' - expected_sql = "SELECT json FROM logger_instance WHERE"\ - " xform_id = %s AND CAST(json->>%s AS INT) > %s AND"\ - " CAST(json->>%s AS INT) < %s AND deleted_at IS NULL"\ - " ORDER BY json->>%s DESC" + expected_sql = ( + "SELECT json FROM logger_instance WHERE" + " xform_id = %s AND CAST(json->>%s AS INT) > %s AND" + " CAST(json->>%s AS INT) < %s AND deleted_at IS NULL" + " ORDER BY json->>%s DESC" + ) (sql, columns, params) = DataView.generate_query_string( self.data_view, @@ -187,13 +184,14 @@ def test_generate_query_string_for_data_with_sort_column_desc(self): self.limit, self.last_submission_time, self.all_data, - sort) + sort, + ) - self.assertEquals(sql, expected_sql) + self.assertEqual(sql, expected_sql) - records = [record for record in DataView.query_iterator(sql, - columns, - params, - self.count)] + records = [ + record + for record in DataView.query_iterator(sql, columns, params, self.count) + ] self.assertTrue(self.is_sorted_desc([r.get("age") for r in records])) diff --git a/onadata/apps/logger/tests/models/test_instance.py b/onadata/apps/logger/tests/models/test_instance.py index 27124f34c3..06741cc3a5 100644 --- a/onadata/apps/logger/tests/models/test_instance.py +++ b/onadata/apps/logger/tests/models/test_instance.py @@ -9,18 +9,23 @@ from onadata.apps.logger.models import XForm, Instance, SubmissionReview from onadata.apps.logger.models.instance import ( - get_id_string_from_xml_str, numeric_checker) + get_id_string_from_xml_str, + numeric_checker, +) from onadata.apps.main.tests.test_base import TestBase -from onadata.apps.viewer.models.parsed_instance import ( - ParsedInstance, query_data) -from onadata.libs.serializers.submission_review_serializer import \ - SubmissionReviewSerializer -from onadata.libs.utils.common_tags import MONGO_STRFTIME, SUBMISSION_TIME, \ - XFORM_ID_STRING, SUBMITTED_BY +from onadata.apps.viewer.models.parsed_instance import ParsedInstance, query_data +from onadata.libs.serializers.submission_review_serializer import ( + SubmissionReviewSerializer, +) +from onadata.libs.utils.common_tags import ( + MONGO_STRFTIME, + SUBMISSION_TIME, + XFORM_ID_STRING, + SUBMITTED_BY, +) class TestInstance(TestBase): - def setUp(self): super(self.__class__, self).setUp() @@ -31,7 +36,7 @@ def test_stores_json(self): for instance in instances: self.assertNotEqual(instance.json, {}) - @patch('django.utils.timezone.now') + @patch("django.utils.timezone.now") def test_json_assigns_attributes(self, mock_time): mock_time.return_value = datetime.utcnow().replace(tzinfo=utc) self._publish_transportation_form_and_submit_instance() @@ -40,12 +45,13 @@ def test_json_assigns_attributes(self, mock_time): instances = Instance.objects.all() for instance in instances: - self.assertEqual(instance.json[SUBMISSION_TIME], - mock_time.return_value.strftime(MONGO_STRFTIME)) - self.assertEqual(instance.json[XFORM_ID_STRING], - xform_id_string) + self.assertEqual( + instance.json[SUBMISSION_TIME], + mock_time.return_value.strftime(MONGO_STRFTIME), + ) + self.assertEqual(instance.json[XFORM_ID_STRING], xform_id_string) - @patch('django.utils.timezone.now') + @patch("django.utils.timezone.now") def test_json_stores_user_attribute(self, mock_time): mock_time.return_value = datetime.utcnow().replace(tzinfo=utc) self._publish_transportation_form() @@ -56,8 +62,13 @@ def test_json_stores_user_attribute(self, mock_time): # submit instance with a request user path = os.path.join( - self.this_directory, 'fixtures', 'transportation', 'instances', - self.surveys[0], self.surveys[0] + '.xml') + self.this_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + self.surveys[0] + ".xml", + ) auth = DigestAuth(self.login_username, self.login_password) self._make_submission(path, auth=auth) @@ -66,19 +77,21 @@ def test_json_stores_user_attribute(self, mock_time): self.assertTrue(len(instances) > 0) for instance in instances: - self.assertEqual(instance.json[SUBMITTED_BY], 'bob') + self.assertEqual(instance.json[SUBMITTED_BY], "bob") # check that the parsed instance's to_dict_for_mongo also contains # the _user key, which is what's used by the JSON REST service pi = ParsedInstance.objects.get(instance=instance) - self.assertEqual(pi.to_dict_for_mongo()[SUBMITTED_BY], 'bob') + self.assertEqual(pi.to_dict_for_mongo()[SUBMITTED_BY], "bob") def test_json_time_match_submission_time(self): self._publish_transportation_form_and_submit_instance() instances = Instance.objects.all() for instance in instances: - self.assertEqual(instance.json[SUBMISSION_TIME], - instance.date_created.strftime(MONGO_STRFTIME)) + self.assertEqual( + instance.json[SUBMISSION_TIME], + instance.date_created.strftime(MONGO_STRFTIME), + ) def test_set_instances_with_geopoints_on_submission_false(self): self._publish_transportation_form() @@ -96,8 +109,7 @@ def test_instances_with_geopoints_in_repeats(self): self.assertFalse(self.xform.instances_with_geopoints) - instance_path = self._fixture_path("gps", - "gps_in_repeats_submission.xml") + instance_path = self._fixture_path("gps", "gps_in_repeats_submission.xml") self._make_submission(instance_path) xform = XForm.objects.get(pk=self.xform.pk) @@ -114,17 +126,17 @@ def test_set_instances_with_geopoints_on_submission_true(self): self.assertTrue(xform.instances_with_geopoints) - @patch('onadata.apps.logger.models.instance.get_values_matching_key') + @patch("onadata.apps.logger.models.instance.get_values_matching_key") def test_instances_with_malformed_geopoints_dont_trigger_value_error( - self, mock_get_values_matching_key): - mock_get_values_matching_key.return_value = '40.81101715564728' + self, mock_get_values_matching_key + ): + mock_get_values_matching_key.return_value = "40.81101715564728" xls_path = self._fixture_path("gps", "gps.xlsx") self._publish_xls_file_and_set_xform(xls_path) self.assertFalse(self.xform.instances_with_geopoints) - path = self._fixture_path( - 'gps', 'instances', 'gps_1980-01-23_20-52-08.xml') + path = self._fixture_path("gps", "instances", "gps_1980-01-23_20-52-08.xml") self._make_submission(path) xform = XForm.objects.get(pk=self.xform.pk) self.assertFalse(xform.instances_with_geopoints) @@ -141,28 +153,25 @@ def test_get_id_string_from_xml_str(self): """ id_string = get_id_string_from_xml_str(submission) - self.assertEqual(id_string, 'id_string') + self.assertEqual(id_string, "id_string") def test_query_data_sort(self): self._publish_transportation_form() self._make_submissions() - latest = Instance.objects.filter(xform=self.xform).latest('pk').pk + latest = Instance.objects.filter(xform=self.xform).latest("pk").pk oldest = Instance.objects.filter(xform=self.xform).first().pk - data = [i.get('_id') for i in query_data( - self.xform, sort='-_id')] + data = [i.get("_id") for i in query_data(self.xform, sort="-_id")] self.assertEqual(data[0], latest) self.assertEqual(data[len(data) - 1], oldest) # sort with a json field - data = [i.get('_id') for i in query_data( - self.xform, sort='{"_id": "-1"}')] + data = [i.get("_id") for i in query_data(self.xform, sort='{"_id": "-1"}')] self.assertEqual(data[0], latest) self.assertEqual(data[len(data) - 1], oldest) # sort with a json field - data = [i.get('_id') for i in query_data( - self.xform, sort='{"_id": -1}')] + data = [i.get("_id") for i in query_data(self.xform, sort='{"_id": -1}')] self.assertEqual(data[0], latest) self.assertEqual(data[len(data) - 1], oldest) @@ -171,45 +180,63 @@ def test_query_filter_by_integer(self): self._make_submissions() oldest = Instance.objects.filter(xform=self.xform).first().pk - data = [i.get('_id') for i in query_data( - self.xform, query='[{"_id": %s}]' % (oldest))] + data = [ + i.get("_id") + for i in query_data(self.xform, query='[{"_id": %s}]' % (oldest)) + ] self.assertEqual(len(data), 1) self.assertEqual(data, [oldest]) # with fields - data = [i.get('_id') for i in query_data( - self.xform, query='{"_id": %s}' % (oldest), fields='["_id"]')] + data = [ + i.get("_id") + for i in query_data( + self.xform, query='{"_id": %s}' % (oldest), fields='["_id"]' + ) + ] self.assertEqual(len(data), 1) - self.assertEqual(data, [oldest]) + self.assertEqual(data, [str(oldest)]) # mongo $gt - data = [i.get('_id') for i in query_data( - self.xform, query='{"_id": {"$gt": %s}}' % (oldest), - fields='["_id"]')] + data = [ + i.get("_id") + for i in query_data( + self.xform, query='{"_id": {"$gt": %s}}' % (oldest), fields='["_id"]' + ) + ] self.assertEqual(self.xform.instances.count(), 4) self.assertEqual(len(data), 3) - @patch('onadata.apps.logger.models.instance.submission_time') + @patch("onadata.apps.logger.models.instance.submission_time") def test_query_filter_by_datetime_field(self, mock_time): self._publish_transportation_form() now = datetime(2014, 1, 1, tzinfo=utc) - times = [now, now + timedelta(seconds=1), now + timedelta(seconds=2), - now + timedelta(seconds=3)] + times = [ + now, + now + timedelta(seconds=1), + now + timedelta(seconds=2), + now + timedelta(seconds=3), + ] mock_time.side_effect = times self._make_submissions() atime = None - for i in self.xform.instances.all().order_by('-pk'): + for i in self.xform.instances.all().order_by("-pk"): i.date_created = times.pop() i.save() if atime is None: atime = i.date_created.strftime(MONGO_STRFTIME) # mongo $gt - data = [i.get('_submission_time') for i in query_data( - self.xform, query='{"_submission_time": {"$lt": "%s"}}' % (atime), - fields='["_submission_time"]')] + data = [ + i.get("_submission_time") + for i in query_data( + self.xform, + query='{"_submission_time": {"$lt": "%s"}}' % (atime), + fields='["_submission_time"]', + ) + ] self.assertEqual(self.xform.instances.count(), 4) self.assertEqual(len(data), 3) self.assertNotIn(atime, data) @@ -224,29 +251,26 @@ def test_instance_json_updated_on_review(self): """ self._publish_transportation_form_and_submit_instance() instance = Instance.objects.first() - self.assertNotIn(u'_review_status', instance.json.keys()) - self.assertNotIn(u'_review_comment', instance.json.keys()) + self.assertNotIn("_review_status", instance.json.keys()) + self.assertNotIn("_review_comment", instance.json.keys()) self.assertFalse(instance.has_a_review) - data = { - "instance": instance.id, - "status": SubmissionReview.APPROVED - } + data = {"instance": instance.id, "status": SubmissionReview.APPROVED} request = HttpRequest() request.user = self.user - serializer_instance = SubmissionReviewSerializer(data=data, context={ - "request": request}) + serializer_instance = SubmissionReviewSerializer( + data=data, context={"request": request} + ) serializer_instance.is_valid() serializer_instance.save() instance.refresh_from_db() instance_review = instance.get_latest_review() - self.assertNotIn(u'_review_comment', instance.json.keys()) - self.assertIn(u'_review_status', instance.json.keys()) - self.assertIn(u'_review_date', instance.json.keys()) - self.assertEqual(SubmissionReview.APPROVED, - instance.json[u'_review_status']) + self.assertNotIn("_review_comment", instance.json.keys()) + self.assertIn("_review_status", instance.json.keys()) + self.assertIn("_review_date", instance.json.keys()) + self.assertEqual(SubmissionReview.APPROVED, instance.json["_review_status"]) self.assertEqual(SubmissionReview.APPROVED, instance_review.status) comment = instance_review.get_note_text() self.assertEqual(None, comment) @@ -255,21 +279,21 @@ def test_instance_json_updated_on_review(self): data = { "instance": instance.id, "note": "Hey There", - "status": SubmissionReview.APPROVED + "status": SubmissionReview.APPROVED, } - serializer_instance = SubmissionReviewSerializer(data=data, context={ - "request": request}) + serializer_instance = SubmissionReviewSerializer( + data=data, context={"request": request} + ) serializer_instance.is_valid() serializer_instance.save() instance.refresh_from_db() instance_review = instance.get_latest_review() - self.assertIn(u'_review_comment', instance.json.keys()) - self.assertIn(u'_review_status', instance.json.keys()) - self.assertIn(u'_review_date', instance.json.keys()) - self.assertEqual(SubmissionReview.APPROVED, - instance.json[u'_review_status']) + self.assertIn("_review_comment", instance.json.keys()) + self.assertIn("_review_status", instance.json.keys()) + self.assertIn("_review_date", instance.json.keys()) + self.assertEqual(SubmissionReview.APPROVED, instance.json["_review_status"]) self.assertEqual(SubmissionReview.APPROVED, instance_review.status) comment = instance_review.get_note_text() self.assertEqual("Hey There", comment) @@ -283,8 +307,8 @@ def test_retrieve_non_existent_submission_review(self): self._publish_transportation_form_and_submit_instance() instance = Instance.objects.first() - self.assertNotIn(u'_review_status', instance.json.keys()) - self.assertNotIn(u'_review_comment', instance.json.keys()) + self.assertNotIn("_review_status", instance.json.keys()) + self.assertNotIn("_review_comment", instance.json.keys()) self.assertFalse(instance.has_a_review) # Update instance has_a_review field @@ -297,9 +321,9 @@ def test_retrieve_non_existent_submission_review(self): self.assertIsNone(instance.get_latest_review()) # Test instance json is not updated - self.assertNotIn(u'_review_comment', instance.json.keys()) - self.assertNotIn(u'_review_status', instance.json.keys()) - self.assertNotIn(u'_review_date', instance.json.keys()) + self.assertNotIn("_review_comment", instance.json.keys()) + self.assertNotIn("_review_status", instance.json.keys()) + self.assertNotIn("_review_date", instance.json.keys()) def test_numeric_checker_with_negative_integer_values(self): # Evaluate negative integer values @@ -318,7 +342,7 @@ def test_numeric_checker_with_negative_integer_values(self): self.assertEqual(result, 36.23) # Evaluate nan values - string_value = float('NaN') + string_value = float("NaN") result = numeric_checker(string_value) self.assertEqual(result, 0) diff --git a/onadata/apps/logger/tests/models/test_note.py b/onadata/apps/logger/tests/models/test_note.py index b6c44412a5..13318d4e90 100644 --- a/onadata/apps/logger/tests/models/test_note.py +++ b/onadata/apps/logger/tests/models/test_note.py @@ -24,5 +24,5 @@ def test_no_created_by(self): ) note.save() note_data = note.get_data() - self.assertEqual(note_data['owner'], "") - self.assertEqual(note_data['created_by'], "") + self.assertEqual(note_data["owner"], "") + self.assertEqual(note_data["created_by"], "") diff --git a/onadata/apps/logger/tests/models/test_submission_review.py b/onadata/apps/logger/tests/models/test_submission_review.py index 00f36a6ec0..ec40421470 100644 --- a/onadata/apps/logger/tests/models/test_submission_review.py +++ b/onadata/apps/logger/tests/models/test_submission_review.py @@ -24,7 +24,7 @@ def test_note_text_property_method(self): instance = Instance.objects.first() note = Note( instance=instance, - note='Hey there', + note="Hey there", instance_field="", ) diff --git a/onadata/apps/logger/tests/models/test_xform.py b/onadata/apps/logger/tests/models/test_xform.py index 856145fc2b..3564f4c100 100644 --- a/onadata/apps/logger/tests/models/test_xform.py +++ b/onadata/apps/logger/tests/models/test_xform.py @@ -5,11 +5,9 @@ import os from builtins import str as text -from past.builtins import basestring # pylint: disable=redefined-builtin from onadata.apps.logger.models import Instance, XForm -from onadata.apps.logger.models.xform import (DuplicateUUIDError, - check_xform_uuid) +from onadata.apps.logger.models.xform import DuplicateUUIDError, check_xform_uuid from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.xform_instance_parser import XLSFormError @@ -18,6 +16,7 @@ class TestXForm(TestBase): """ Test XForm model. """ + def test_submission_count(self): """ Test submission count does not include deleted submissions. @@ -43,14 +42,17 @@ def test_set_title_unicode_error(self): """ xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../..", "fixtures", "tutorial", "tutorial_arabic_labels.xlsx" + "../..", + "fixtures", + "tutorial", + "tutorial_arabic_labels.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) - self.assertTrue(isinstance(self.xform.xml, basestring)) + self.assertTrue(isinstance(self.xform.xml, str)) # change title - self.xform.title = u'Random Title' + self.xform.title = "Random Title" self.assertNotIn(self.xform.title, self.xform.xml) @@ -65,7 +67,7 @@ def test_version_length(self): """Test Xform.version can store more than 12 chars""" self._publish_transportation_form_and_submit_instance() xform = XForm.objects.get(pk=self.xform.id) - xform.version = u'12345678901234567890' + xform.version = "12345678901234567890" xform.save() self.assertTrue(len(xform.version) > 12) @@ -89,7 +91,7 @@ def test_soft_delete(self): # '&' should raise an XLSFormError exception when being changed, for # deletions this should not raise any exception however - xform.title = 'Trial & Error' + xform.title = "Trial & Error" xform.soft_delete(self.user) xform.refresh_from_db() @@ -103,7 +105,7 @@ def test_soft_delete(self): # deleted-at suffix is present self.assertIn("-deleted-at-", xform.id_string) self.assertIn("-deleted-at-", xform.sms_id_string) - self.assertEqual(xform.deleted_by.username, 'bob') + self.assertEqual(xform.deleted_by.username, "bob") def test_get_survey_element(self): """ @@ -127,7 +129,7 @@ def test_get_survey_element(self): | | fruity | orange | Orange | | | fruity | mango | Mango | """ - kwargs = {'name': 'favs', 'title': 'Fruits', 'id_string': 'favs'} + kwargs = {"name": "favs", "title": "Fruits", "id_string": "favs"} survey = self.md_to_pyxform_survey(markdown_xlsform, kwargs) xform = XForm() xform._survey = survey # pylint: disable=W0212 @@ -136,7 +138,7 @@ def test_get_survey_element(self): self.assertIsNone(xform.get_survey_element("non_existent")) # get fruita element by name - fruita = xform.get_survey_element('fruita') + fruita = xform.get_survey_element("fruita") self.assertEqual(fruita.get_abbreviated_xpath(), "a/fruita") # get exact choices element from choice abbreviated xpath @@ -149,7 +151,7 @@ def test_get_survey_element(self): fruitb_o = xform.get_survey_element("b/fruitb/orange") self.assertEqual(fruitb_o.get_abbreviated_xpath(), "b/fruitb/orange") - self.assertEqual(xform.get_child_elements('NoneExistent'), []) + self.assertEqual(xform.get_child_elements("NoneExistent"), []) def test_check_xform_uuid(self): """ @@ -173,8 +175,10 @@ def test_id_string_max_length_on_soft_delete(self): """ self._publish_transportation_form_and_submit_instance() xform = XForm.objects.get(pk=self.xform.id) - new_string = "transportation_twenty_fifth_july_two_thousand_and_" \ - "eleven_test_for_long_sms_id_string_and_id_string" + new_string = ( + "transportation_twenty_fifth_july_two_thousand_and_" + "eleven_test_for_long_sms_id_string_and_id_string" + ) xform.id_string = new_string xform.sms_id_string = new_string @@ -187,15 +191,13 @@ def test_id_string_max_length_on_soft_delete(self): # '&' should raise an XLSFormError exception when being changed, for # deletions this should not raise any exception however - xform.title = 'Trial & Error' + xform.title = "Trial & Error" xform.soft_delete(self.user) xform.refresh_from_db() - d_id_string = new_string + xform.deleted_at.strftime( - '-deleted-at-%s') - d_sms_id_string = new_string + xform.deleted_at.strftime( - '-deleted-at-%s') + d_id_string = new_string + xform.deleted_at.strftime("-deleted-at-%s") + d_sms_id_string = new_string + xform.deleted_at.strftime("-deleted-at-%s") # deleted_at is not None self.assertIsNotNone(xform.deleted_at) @@ -209,15 +211,17 @@ def test_id_string_max_length_on_soft_delete(self): self.assertIn(xform.sms_id_string, d_sms_id_string) self.assertEqual(xform.id_string, d_id_string[:100]) self.assertEqual(xform.sms_id_string, d_sms_id_string[:100]) - self.assertEqual(xform.deleted_by.username, 'bob') + self.assertEqual(xform.deleted_by.username, "bob") def test_id_string_length(self): """Test Xform.id_string cannot store more than 100 chars""" self._publish_transportation_form_and_submit_instance() xform = XForm.objects.get(pk=self.xform.id) - new_string = "transportation_twenty_fifth_july_two_thousand_and_" \ - "eleven_test_for_long_sms_id_string_and_id_string_" \ - "before_save" + new_string = ( + "transportation_twenty_fifth_july_two_thousand_and_" + "eleven_test_for_long_sms_id_string_and_id_string_" + "before_save" + ) xform.id_string = new_string xform.sms_id_string = new_string diff --git a/onadata/apps/logger/tests/test_backup_tools.py b/onadata/apps/logger/tests/test_backup_tools.py index e9509e9fb0..eeebcd9679 100644 --- a/onadata/apps/logger/tests/test_backup_tools.py +++ b/onadata/apps/logger/tests/test_backup_tools.py @@ -9,8 +9,11 @@ from onadata.apps.logger.models import Instance from onadata.apps.logger.import_tools import django_file from onadata.apps.logger.import_tools import create_instance -from onadata.libs.utils.backup_tools import _date_created_from_filename,\ - create_zip_backup, restore_backup_from_zip +from onadata.libs.utils.backup_tools import ( + _date_created_from_filename, + create_zip_backup, + restore_backup_from_zip, +) class TestBackupTools(TestBase): @@ -19,8 +22,13 @@ def setUp(self): self._publish_xls_file_and_set_xform( os.path.join( settings.PROJECT_ROOT, - "apps", "logger", "fixtures", "test_forms", - "tutorial.xlsx")) + "apps", + "logger", + "fixtures", + "test_forms", + "tutorial.xlsx", + ) + ) def test_date_created_override(self): """ @@ -28,19 +36,30 @@ def test_date_created_override(self): will set our date as the date_created """ xml_file_path = os.path.join( - settings.PROJECT_ROOT, "apps", "logger", "fixtures", - "tutorial", "instances", "tutorial_2012-06-27_11-27-53.xml") + settings.PROJECT_ROOT, + "apps", + "logger", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53.xml", + ) xml_file = django_file( - xml_file_path, field_name="xml_file", content_type="text/xml") + xml_file_path, field_name="xml_file", content_type="text/xml" + ) media_files = [] - date_created = datetime.strptime("2013-01-01 12:00:00", - "%Y-%m-%d %H:%M:%S") + date_created = datetime.strptime("2013-01-01 12:00:00", "%Y-%m-%d %H:%M:%S") instance = create_instance( - self.user.username, xml_file, media_files, - date_created_override=date_created) + self.user.username, + xml_file, + media_files, + date_created_override=date_created, + ) self.assertIsNotNone(instance) - self.assertEqual(instance.date_created.strftime("%Y-%m-%d %H:%M:%S"), - date_created.strftime("%Y-%m-%d %H:%M:%S")) + self.assertEqual( + instance.date_created.strftime("%Y-%m-%d %H:%M:%S"), + date_created.strftime("%Y-%m-%d %H:%M:%S"), + ) def test_date_created_from_filename(self): date_str = "2012-01-02-12-35-48" @@ -55,17 +74,14 @@ def test_date_created_from_filename(self): def test_backup_then_restore_from_zip(self): self._publish_transportation_form() - initial_instance_count = Instance.objects.filter( - xform=self.xform).count() + initial_instance_count = Instance.objects.filter(xform=self.xform).count() # make submissions for i in range(len(self.surveys)): self._submit_transport_instance(i) - instance_count = Instance.objects.filter( - xform=self.xform).count() - self.assertEqual( - instance_count, initial_instance_count + len(self.surveys)) + instance_count = Instance.objects.filter(xform=self.xform).count() + self.assertEqual(instance_count, initial_instance_count + len(self.surveys)) # make a backup temp_dir = tempfile.mkdtemp() @@ -77,17 +93,14 @@ def test_backup_then_restore_from_zip(self): for instance in Instance.objects.filter(xform=self.xform): instance.delete() - instance_count = Instance.objects.filter( - xform=self.xform).count() + instance_count = Instance.objects.filter(xform=self.xform).count() # remove temp dir tree self.assertEqual(instance_count, 0) # restore instances self.assertTrue(os.path.exists(zip_file.name)) - restore_backup_from_zip( - zip_file.name, self.user.username) - instance_count = Instance.objects.filter( - xform=self.xform).count() + restore_backup_from_zip(zip_file.name, self.user.username) + instance_count = Instance.objects.filter(xform=self.xform).count() # remove temp dir tree self.assertEqual(instance_count, len(self.surveys)) shutil.rmtree(temp_dir) diff --git a/onadata/apps/logger/tests/test_briefcase_api.py b/onadata/apps/logger/tests/test_briefcase_api.py index 1dca42662d..5077093359 100644 --- a/onadata/apps/logger/tests/test_briefcase_api.py +++ b/onadata/apps/logger/tests/test_briefcase_api.py @@ -21,33 +21,35 @@ def ordered_instances(xform): - return Instance.objects.filter(xform=xform).order_by('id') + return Instance.objects.filter(xform=xform).order_by("id") class TestBriefcaseAPI(TestBase): - def setUp(self): super(TestBase, self).setUp() self.factory = APIRequestFactory() self._create_user_and_login() self._logout() self.form_def_path = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'transportation.xml') + self.this_directory, "fixtures", "transportation", "transportation.xml" + ) self._submission_list_url = reverse( - 'view-submission-list', kwargs={'username': self.user.username}) + "view-submission-list", kwargs={"username": self.user.username} + ) self._submission_url = reverse( - 'submissions', kwargs={'username': self.user.username}) + "submissions", kwargs={"username": self.user.username} + ) self._download_submission_url = reverse( - 'view-download-submission', - kwargs={'username': self.user.username}) + "view-download-submission", kwargs={"username": self.user.username} + ) self._form_upload_url = reverse( - 'form-upload', kwargs={'username': self.user.username}) + "form-upload", kwargs={"username": self.user.username} + ) def test_view_submission_list(self): self._publish_xml_form() self._make_submissions() - params = {'formId': self.xform.id_string} + params = {"formId": self.xform.id_string} request = self.factory.get(self._submission_list_url, params) response = view_submission_list(request, self.user.username) self.assertEqual(response.status_code, 401) @@ -56,27 +58,30 @@ def test_view_submission_list(self): response = view_submission_list(request, self.user.username) self.assertEqual(response.status_code, 200) submission_list_path = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'view', 'submissionList.xml') + self.this_directory, + "fixtures", + "transportation", + "view", + "submissionList.xml", + ) instances = ordered_instances(self.xform) self.assertEqual(instances.count(), NUM_INSTANCES) last_index = instances[instances.count() - 1].pk - with open(submission_list_path, encoding='utf-8') as f: + with open(submission_list_path, encoding="utf-8") as f: expected_submission_list = f.read() - expected_submission_list = \ - expected_submission_list.replace( - '{{resumptionCursor}}', '%s' % last_index) - self.assertEqual(response.content.decode('utf-8'), - expected_submission_list) + expected_submission_list = expected_submission_list.replace( + "{{resumptionCursor}}", "%s" % last_index + ) + self.assertEqual(response.content.decode("utf-8"), expected_submission_list) def test_view_submission_list_w_deleted_submission(self): self._publish_xml_form() self._make_submissions() - uuid = 'f3d8dc65-91a6-4d0f-9e97-802128083390' - Instance.objects.filter(uuid=uuid).order_by('id').delete() - params = {'formId': self.xform.id_string} + uuid = "f3d8dc65-91a6-4d0f-9e97-802128083390" + Instance.objects.filter(uuid=uuid).order_by("id").delete() + params = {"formId": self.xform.id_string} request = self.factory.get(self._submission_list_url, params) response = view_submission_list(request, self.user.username) self.assertEqual(response.status_code, 401) @@ -85,39 +90,43 @@ def test_view_submission_list_w_deleted_submission(self): response = view_submission_list(request, self.user.username) self.assertEqual(response.status_code, 200) submission_list_path = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'view', 'submissionList-4.xml') + self.this_directory, + "fixtures", + "transportation", + "view", + "submissionList-4.xml", + ) instances = ordered_instances(self.xform) self.assertEqual(instances.count(), NUM_INSTANCES - 1) last_index = instances[instances.count() - 1].pk - with open(submission_list_path, encoding='utf-8') as f: + with open(submission_list_path, encoding="utf-8") as f: expected_submission_list = f.read() - expected_submission_list = \ - expected_submission_list.replace( - '{{resumptionCursor}}', '%s' % last_index) - self.assertEqual(response.content.decode('utf-8'), - expected_submission_list) - - formId = u'%(formId)s[@version=null and @uiVersion=null]/' \ - u'%(formId)s[@key=uuid:%(instanceId)s]' % { - 'formId': self.xform.id_string, - 'instanceId': uuid} - params = {'formId': formId} + expected_submission_list = expected_submission_list.replace( + "{{resumptionCursor}}", "%s" % last_index + ) + self.assertEqual(response.content.decode("utf-8"), expected_submission_list) + + formId = ( + "%(formId)s[@version=null and @uiVersion=null]/" + "%(formId)s[@key=uuid:%(instanceId)s]" + % {"formId": self.xform.id_string, "instanceId": uuid} + ) + params = {"formId": formId} response = self.client.get(self._download_submission_url, data=params) self.assertTrue(response.status_code, 404) def test_view_submission_list_OtherUser(self): self._publish_xml_form() self._make_submissions() - params = {'formId': self.xform.id_string} + params = {"formId": self.xform.id_string} # deno cannot view bob's submissionList - self._create_user('deno', 'deno') + self._create_user("deno", "deno") request = self.factory.get(self._submission_list_url, params) response = view_submission_list(request, self.user.username) self.assertEqual(response.status_code, 401) - auth = DigestAuth('deno', 'deno') + auth = DigestAuth("deno", "deno") request.META.update(auth(request.META, response)) response = view_submission_list(request, self.user.username) self.assertEqual(response.status_code, 403) @@ -137,8 +146,8 @@ def get_last_index(xform, last_index=None): self._publish_xml_form() self._make_submissions() - params = {'formId': self.xform.id_string} - params['numEntries'] = 2 + params = {"formId": self.xform.id_string} + params["numEntries"] = 2 instances = ordered_instances(self.xform) self.assertEqual(instances.count(), NUM_INSTANCES) @@ -156,37 +165,42 @@ def get_last_index(xform, last_index=None): if index > 2: last_index = get_last_index(self.xform, last_index) - filename = 'submissionList-%s.xml' % index + filename = "submissionList-%s.xml" % index if index == 4: - self.assertEqual(response.content.decode('utf-8'), - last_expected_submission_list) + self.assertEqual( + response.content.decode("utf-8"), last_expected_submission_list + ) continue # set cursor for second request - params['cursor'] = last_index + params["cursor"] = last_index submission_list_path = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'view', filename) - with open(submission_list_path, encoding='utf-8') as f: + self.this_directory, "fixtures", "transportation", "view", filename + ) + with open(submission_list_path, encoding="utf-8") as f: expected_submission_list = f.read() - last_expected_submission_list = expected_submission_list = \ - expected_submission_list.replace( - '{{resumptionCursor}}', '%s' % last_index) - self.assertEqual(response.content.decode('utf-8'), - expected_submission_list) + last_expected_submission_list = ( + expected_submission_list + ) = expected_submission_list.replace( + "{{resumptionCursor}}", "%s" % last_index + ) + self.assertEqual( + response.content.decode("utf-8"), expected_submission_list + ) last_index += 2 def test_view_downloadSubmission(self): self._publish_xml_form() self.maxDiff = None self._submit_transport_instance_w_attachment() - instanceId = u'5b2cc313-fc09-437e-8149-fcd32f695d41' + instanceId = "5b2cc313-fc09-437e-8149-fcd32f695d41" instance = Instance.objects.get(uuid=instanceId) - formId = u'%(formId)s[@version=null and @uiVersion=null]/' \ - u'%(formId)s[@key=uuid:%(instanceId)s]' % { - 'formId': self.xform.id_string, - 'instanceId': instanceId} - params = {'formId': formId} + formId = ( + "%(formId)s[@version=null and @uiVersion=null]/" + "%(formId)s[@key=uuid:%(instanceId)s]" + % {"formId": self.xform.id_string, "instanceId": instanceId} + ) + params = {"formId": formId} request = self.factory.get(self._download_submission_url, params) response = view_download_submission(request, self.user.username) self.assertEqual(response.status_code, 401) @@ -195,49 +209,55 @@ def test_view_downloadSubmission(self): response = view_download_submission(request, self.user.username) text = "uuid:%s" % instanceId download_submission_path = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'view', 'downloadSubmission.xml') - with open(download_submission_path, encoding='utf-8') as f: + self.this_directory, + "fixtures", + "transportation", + "view", + "downloadSubmission.xml", + ) + with open(download_submission_path, encoding="utf-8") as f: text = f.read() - for var in ((u'{{submissionDate}}', - instance.date_created.isoformat()), - (u'{{form_id}}', str(self.xform.id))): + for var in ( + ("{{submissionDate}}", instance.date_created.isoformat()), + ("{{form_id}}", str(self.xform.id)), + ): text = text.replace(*var) self.assertContains(response, instanceId, status_code=200) - self.assertMultiLineEqual(response.content.decode('utf-8'), text) + self.assertMultiLineEqual(response.content.decode("utf-8"), text) def test_view_downloadSubmission_OtherUser(self): self._publish_xml_form() self.maxDiff = None self._submit_transport_instance_w_attachment() - instanceId = u'5b2cc313-fc09-437e-8149-fcd32f695d41' - formId = u'%(formId)s[@version=null and @uiVersion=null]/' \ - u'%(formId)s[@key=uuid:%(instanceId)s]' % { - 'formId': self.xform.id_string, - 'instanceId': instanceId} - params = {'formId': formId} + instanceId = "5b2cc313-fc09-437e-8149-fcd32f695d41" + formId = ( + "%(formId)s[@version=null and @uiVersion=null]/" + "%(formId)s[@key=uuid:%(instanceId)s]" + % {"formId": self.xform.id_string, "instanceId": instanceId} + ) + params = {"formId": formId} # deno cannot view bob's downloadSubmission - self._create_user('deno', 'deno') + self._create_user("deno", "deno") request = self.factory.get(self._download_submission_url, params) response = view_download_submission(request, self.user.username) self.assertEqual(response.status_code, 401) - auth = DigestAuth('deno', 'deno') + auth = DigestAuth("deno", "deno") request.META.update(auth(request.META, response)) response = view_download_submission(request, self.user.username) self.assertEqual(response.status_code, 403) def test_publish_xml_form_OtherUser(self): # deno cannot publish form to bob's account - self._create_user('deno', 'deno') + self._create_user("deno", "deno") count = XForm.objects.count() - with open(self.form_def_path, encoding='utf-8') as f: - params = {'form_def_file': f, 'dataFile': ''} + with open(self.form_def_path, encoding="utf-8") as f: + params = {"form_def_file": f, "dataFile": ""} request = self.factory.post(self._form_upload_url, params) response = form_upload(request, username=self.user.username) self.assertEqual(response.status_code, 401) - auth = DigestAuth('deno', 'deno') + auth = DigestAuth("deno", "deno") request.META.update(auth(request.META, response)) response = form_upload(request, username=self.user.username) self.assertNotEqual(XForm.objects.count(), count + 1) @@ -245,40 +265,38 @@ def test_publish_xml_form_OtherUser(self): def test_publish_xml_form_where_filename_is_not_id_string(self): form_def_path = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'Transportation Form.xml') + self.this_directory, "fixtures", "transportation", "Transportation Form.xml" + ) count = XForm.objects.count() - with open(form_def_path, encoding='utf-8') as f: - params = {'form_def_file': f, 'dataFile': ''} + with open(form_def_path, encoding="utf-8") as f: + params = {"form_def_file": f, "dataFile": ""} request = self.factory.post(self._form_upload_url, params) response = form_upload(request, username=self.user.username) self.assertEqual(response.status_code, 401) auth = DigestAuth(self.login_username, self.login_password) request.META.update(auth(request.META, response)) response = form_upload(request, username=self.user.username) - self.assertContains( - response, "successfully published.", status_code=201) + self.assertContains(response, "successfully published.", status_code=201) self.assertEqual(XForm.objects.count(), count + 1) def _publish_xml_form(self): count = XForm.objects.count() - with open(self.form_def_path, encoding='utf-8') as f: - params = {'form_def_file': f, 'dataFile': ''} + with open(self.form_def_path, encoding="utf-8") as f: + params = {"form_def_file": f, "dataFile": ""} request = self.factory.post(self._form_upload_url, params) response = form_upload(request, username=self.user.username) self.assertEqual(response.status_code, 401) auth = DigestAuth(self.login_username, self.login_password) request.META.update(auth(request.META, response)) response = form_upload(request, username=self.user.username) - self.assertContains( - response, "successfully published.", status_code=201) + self.assertContains(response, "successfully published.", status_code=201) self.assertEqual(XForm.objects.count(), count + 1) - self.xform = XForm.objects.order_by('pk').reverse()[0] + self.xform = XForm.objects.order_by("pk").reverse()[0] def test_form_upload(self): self._publish_xml_form() - with open(self.form_def_path, encoding='utf-8') as f: - params = {'form_def_file': f, 'dataFile': ''} + with open(self.form_def_path, encoding="utf-8") as f: + params = {"form_def_file": f, "dataFile": ""} request = self.factory.post(self._form_upload_url, params) response = form_upload(request, username=self.user.username) self.assertEqual(response.status_code, 401) @@ -287,25 +305,24 @@ def test_form_upload(self): response = form_upload(request, username=self.user.username) self.assertContains( response, - u'Form with this id or SMS-keyword already exists', - status_code=400) + "Form with this id or SMS-keyword already exists", + status_code=400, + ) def test_submission_with_instance_id_on_root_node(self): self._publish_xml_form() - message = u"Successful submission." - instanceId = u'5b2cc313-fc09-437e-8149-fcd32f695d41' - self.assertRaises( - Instance.DoesNotExist, Instance.objects.get, uuid=instanceId) + message = "Successful submission." + instanceId = "5b2cc313-fc09-437e-8149-fcd32f695d41" + self.assertRaises(Instance.DoesNotExist, Instance.objects.get, uuid=instanceId) submission_path = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'view', 'submission.xml') + self.this_directory, "fixtures", "transportation", "view", "submission.xml" + ) count = Instance.objects.count() - with open(submission_path, encoding='utf-8') as f: - post_data = {'xml_submission_file': f} + with open(submission_path, encoding="utf-8") as f: + post_data = {"xml_submission_file": f} self.factory = APIRequestFactory() request = self.factory.post(self._submission_url, post_data) - request.user = authenticate(username='bob', - password='bob') + request.user = authenticate(username="bob", password="bob") response = submission(request, username=self.user.username) self.assertContains(response, message, status_code=201) self.assertContains(response, instanceId, status_code=201) diff --git a/onadata/apps/logger/tests/test_briefcase_client.py b/onadata/apps/logger/tests/test_briefcase_client.py index 58f7cfcc57..2bf42c9663 100644 --- a/onadata/apps/logger/tests/test_briefcase_client.py +++ b/onadata/apps/logger/tests/test_briefcase_client.py @@ -10,7 +10,7 @@ from django.test import RequestFactory from django.urls import reverse from django_digest.test import Client as DigestClient -from future.moves.urllib.parse import urljoin +from six.moves.urllib.parse import urljoin from httmock import HTTMock, urlmatch from onadata.apps.logger.models import Instance, XForm @@ -23,35 +23,45 @@ storage = get_storage_class()() -@urlmatch(netloc=r'(.*\.)?testserver$') +@urlmatch(netloc=r"(.*\.)?testserver$") def form_list_xml(url, request, **kwargs): + """Mock different ODK Aggregate Server API requests.""" response = requests.Response() factory = RequestFactory() req = factory.get(url.path) - req.user = authenticate(username='bob', password='bob') + req.user = authenticate(username="bob", password="bob") req.user.profile.require_auth = False req.user.profile.save() - id_string = 'transportation_2011_07_25' - if url.path.endswith('formList'): - res = formList(req, username='bob') - elif url.path.endswith('form.xml'): - res = download_xform(req, username='bob', id_string=id_string) - elif url.path.find('xformsManifest') > -1: - res = xformsManifest(req, username='bob', id_string=id_string) - elif url.path.find('formid-media') > -1: - data_id = url.path[url.path.rfind('/') + 1:] + id_string = "transportation_2011_07_25" + if url.path.endswith("formList"): + res = formList(req, username="bob") + elif url.path.endswith("form.xml"): + res = download_xform(req, username="bob", id_string=id_string) + elif url.path.find("xformsManifest") > -1: + res = xformsManifest(req, username="bob", id_string=id_string) + elif url.path.find("formid-media") > -1: + data_id = url.path[url.path.rfind("/") + 1 :] res = download_media_data( - req, username='bob', id_string=id_string, data_id=data_id) + req, username="bob", id_string=id_string, data_id=data_id + ) + ids = list(Instance.objects.values_list("id", flat=True)) + xids = list(XForm.objects.values_list("id", flat=True)) + assert ( + res.status_code == 200 + ), f"{data_id} - {res.content} {res.status_code} -{ids} {xids} {url}" + # pylint: disable=protected-access response._content = get_streaming_content(res) else: - res = formList(req, username='bob') + res = formList(req, username="bob") response.status_code = 200 if not response._content: + # pylint: disable=protected-access response._content = res.content return response def get_streaming_content(res): + """Return the contents of ``res.streaming_content``.""" tmp = BytesIO() for chunk in res.streaming_content: tmp.write(chunk) @@ -60,15 +70,16 @@ def get_streaming_content(res): return content -@urlmatch(netloc=r'(.*\.)?testserver$') +@urlmatch(netloc=r"(.*\.)?testserver$") def instances_xml(url, request, **kwargs): response = requests.Response() client = DigestClient() - client.set_authorization('bob', 'bob', 'Digest') - res = client.get('%s?%s' % (url.path, url.query)) + client.set_authorization("bob", "bob", "Digest") + res = client.get(f"{url.path}?{url.query}") if res.status_code == 302: - res = client.get(res['Location']) - response.encoding = res.get('content-type') + res = client.get(res["Location"]) + assert res.status_code == 200, res.content + response.encoding = res.get("content-type") response._content = get_streaming_content(res) else: response._content = res.content @@ -77,27 +88,24 @@ def instances_xml(url, request, **kwargs): class TestBriefcaseClient(TestBase): - def setUp(self): TestBase.setUp(self) self._publish_transportation_form() self._submit_transport_instance_w_attachment() - src = os.path.join(self.this_directory, "fixtures", - "transportation", "screenshot.png") - uf = UploadedFile(file=open(src, 'rb'), content_type='image/png') + src = os.path.join( + self.this_directory, "fixtures", "transportation", "screenshot.png" + ) + uf = UploadedFile(file=open(src, "rb"), content_type="image/png") count = MetaData.objects.count() MetaData.media_upload(self.xform, uf) self.assertEqual(MetaData.objects.count(), count + 1) url = urljoin( - self.base_url, - reverse(profile, kwargs={'username': self.user.username}) + self.base_url, reverse(profile, kwargs={"username": self.user.username}) ) self._logout() - self._create_user_and_login('deno', 'deno') + self._create_user_and_login("deno", "deno") self.bc = BriefcaseClient( - username='bob', password='bob', - url=url, - user=self.user + username="bob", password="bob", url=url, user=self.user ) def test_download_xform_xml(self): @@ -107,31 +115,31 @@ def test_download_xform_xml(self): with HTTMock(form_list_xml): self.bc.download_xforms() forms_folder_path = os.path.join( - 'deno', 'briefcase', 'forms', self.xform.id_string) + "deno", "briefcase", "forms", self.xform.id_string + ) self.assertTrue(storage.exists(forms_folder_path)) - forms_path = os.path.join(forms_folder_path, - '%s.xml' % self.xform.id_string) + forms_path = os.path.join(forms_folder_path, f"{self.xform.id_string}.xml") self.assertTrue(storage.exists(forms_path)) - form_media_path = os.path.join(forms_folder_path, 'form-media') + form_media_path = os.path.join(forms_folder_path, "form-media") self.assertTrue(storage.exists(form_media_path)) - media_path = os.path.join(form_media_path, 'screenshot.png') + media_path = os.path.join(form_media_path, "screenshot.png") self.assertTrue(storage.exists(media_path)) - """ - Download instance xml - """ with HTTMock(instances_xml): self.bc.download_instances(self.xform.id_string) instance_folder_path = os.path.join( - 'deno', 'briefcase', 'forms', self.xform.id_string, 'instances') + "deno", "briefcase", "forms", self.xform.id_string, "instances" + ) self.assertTrue(storage.exists(instance_folder_path)) instance = Instance.objects.all()[0] instance_path = os.path.join( - instance_folder_path, 'uuid%s' % instance.uuid, 'submission.xml') + instance_folder_path, f"uuid{instance.uuid}", "submission.xml" + ) self.assertTrue(storage.exists(instance_path)) media_file = "1335783522563.jpg" media_path = os.path.join( - instance_folder_path, 'uuid%s' % instance.uuid, media_file) + instance_folder_path, f"uuid{instance.uuid}", media_file + ) self.assertTrue(storage.exists(media_path)) def test_push(self): @@ -140,22 +148,22 @@ def test_push(self): with HTTMock(instances_xml): self.bc.download_instances(self.xform.id_string) XForm.objects.all().delete() - xforms = XForm.objects.filter( - user=self.user, id_string=self.xform.id_string) + xforms = XForm.objects.filter(user=self.user, id_string=self.xform.id_string) self.assertEqual(xforms.count(), 0) instances = Instance.objects.filter( - xform__user=self.user, xform__id_string=self.xform.id_string) + xform__user=self.user, xform__id_string=self.xform.id_string + ) self.assertEqual(instances.count(), 0) self.bc.push() - xforms = XForm.objects.filter( - user=self.user, id_string=self.xform.id_string) + xforms = XForm.objects.filter(user=self.user, id_string=self.xform.id_string) self.assertEqual(xforms.count(), 1) instances = Instance.objects.filter( - xform__user=self.user, xform__id_string=self.xform.id_string) + xform__user=self.user, xform__id_string=self.xform.id_string + ) self.assertEqual(instances.count(), 1) def tearDown(self): # remove media files - for username in ['bob', 'deno']: + for username in ["bob", "deno"]: if storage.exists(username): shutil.rmtree(storage.path(username)) diff --git a/onadata/apps/logger/tests/test_digest_authentication.py b/onadata/apps/logger/tests/test_digest_authentication.py index 3d0b594461..421059c165 100644 --- a/onadata/apps/logger/tests/test_digest_authentication.py +++ b/onadata/apps/logger/tests/test_digest_authentication.py @@ -14,7 +14,7 @@ from onadata.apps.api.models.odk_token import ODKToken -ODK_TOKEN_STORAGE = 'onadata.apps.api.storage.ODKTokenAccountStorage' +ODK_TOKEN_STORAGE = "onadata.apps.api.storage.ODKTokenAccountStorage" class TestDigestAuthentication(TestBase): @@ -29,45 +29,49 @@ def test_authenticated_submissions(self): """ s = self.surveys[0] xml_submission_file_path = os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml' + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", ) self._set_require_auth() auth = DigestAuth(self.login_username, self.login_password) - self._make_submission(xml_submission_file_path, add_uuid=True, - auth=auth) + self._make_submission(xml_submission_file_path, add_uuid=True, auth=auth) self.assertEqual(self.response.status_code, 201) def _set_require_auth(self, auth=True): - profile, created = \ - UserProfile.objects.get_or_create(user=self.user) + profile, created = UserProfile.objects.get_or_create(user=self.user) profile.require_auth = auth profile.save() def test_fail_authenticated_submissions_to_wrong_account(self): - username = 'dennis' + username = "dennis" # set require_auth b4 we switch user self._set_require_auth() self._create_user_and_login(username=username, password=username) self._set_require_auth() s = self.surveys[0] xml_submission_file_path = os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml' + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", ) - self._make_submission(xml_submission_file_path, add_uuid=True, - auth=DigestAuth('alice', 'alice')) + self._make_submission( + xml_submission_file_path, add_uuid=True, auth=DigestAuth("alice", "alice") + ) # Authentication required self.assertEqual(self.response.status_code, 401) - auth = DigestAuth('dennis', 'dennis') + auth = DigestAuth("dennis", "dennis") with self.assertRaises(Http404): - self._make_submission(xml_submission_file_path, add_uuid=True, - auth=auth) + self._make_submission(xml_submission_file_path, add_uuid=True, auth=auth) - @override_settings( - DIGEST_ACCOUNT_BACKEND=ODK_TOKEN_STORAGE - ) + @override_settings(DIGEST_ACCOUNT_BACKEND=ODK_TOKEN_STORAGE) def test_digest_authentication_with_odk_token_storage(self): """ Test that a valid Digest request with as the auth email:odk_token @@ -75,42 +79,45 @@ def test_digest_authentication_with_odk_token_storage(self): """ s = self.surveys[0] xml_submission_file_path = os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml' + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", ) self._set_require_auth() # Set email for user - self.user.email = 'bob@bob.test' + self.user.email = "bob@bob.test" self.user.save() odk_token = ODKToken.objects.create(user=self.user) # The value odk_token.key is hashed we need to have the raw_key # In order to authenticate with DigestAuth - fernet = Fernet(getattr(settings, 'ODK_TOKEN_FERNET_KEY')) - raw_key = fernet.decrypt(odk_token.key.encode('utf-8')).decode('utf-8') + fernet = Fernet(getattr(settings, "ODK_TOKEN_FERNET_KEY")) + raw_key = fernet.decrypt(odk_token.key.encode("utf-8")).decode("utf-8") auth = DigestAuth(self.user.email, raw_key) - self._make_submission(xml_submission_file_path, add_uuid=True, - auth=auth) + self._make_submission(xml_submission_file_path, add_uuid=True, auth=auth) self.assertEqual(self.response.status_code, 201) # Test can authenticate with username:token s = self.surveys[1] xml_submission_file_path = os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml' + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", ) auth = DigestAuth(self.user.username, raw_key) - self._make_submission(xml_submission_file_path, add_uuid=True, - auth=auth) + self._make_submission(xml_submission_file_path, add_uuid=True, auth=auth) self.assertEqual(self.response.status_code, 201) - @override_settings( - ODK_KEY_LIFETIME=1, - DIGEST_ACCOUNT_BACKEND=ODK_TOKEN_STORAGE - ) + @override_settings(ODK_KEY_LIFETIME=1, DIGEST_ACCOUNT_BACKEND=ODK_TOKEN_STORAGE) def test_fails_authentication_past_odk_token_expiry(self): """ Test that a Digest authenticated request using an ODK Token that has @@ -118,13 +125,17 @@ def test_fails_authentication_past_odk_token_expiry(self): """ s = self.surveys[0] xml_submission_file_path = os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml' + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", ) self._set_require_auth() # Set email for user - self.user.email = 'bob@bob.test' + self.user.email = "bob@bob.test" self.user.save() odk_token = ODKToken.objects.create(user=self.user) @@ -134,10 +145,9 @@ def test_fails_authentication_past_odk_token_expiry(self): # The value odk_token.key is hashed we need to have the raw_key # In order to authenticate with DigestAuth - fernet = Fernet(getattr(settings, 'ODK_TOKEN_FERNET_KEY')) - raw_key = fernet.decrypt(odk_token.key.encode('utf-8')).decode('utf-8') + fernet = Fernet(getattr(settings, "ODK_TOKEN_FERNET_KEY")) + raw_key = fernet.decrypt(odk_token.key.encode("utf-8")).decode("utf-8") auth = DigestAuth(self.user.email, raw_key) - self._make_submission(xml_submission_file_path, add_uuid=True, - auth=auth) + self._make_submission(xml_submission_file_path, add_uuid=True, auth=auth) self.assertEqual(self.response.status_code, 401) diff --git a/onadata/apps/logger/tests/test_encrypted_submissions.py b/onadata/apps/logger/tests/test_encrypted_submissions.py index ce85c0a86a..80d9f7809a 100644 --- a/onadata/apps/logger/tests/test_encrypted_submissions.py +++ b/onadata/apps/logger/tests/test_encrypted_submissions.py @@ -25,38 +25,47 @@ def setUp(self): super(TestEncryptedForms, self).setUp() self._create_user_and_login() self._submission_url = reverse( - 'submissions', kwargs={'username': self.user.username}) + "submissions", kwargs={"username": self.user.username} + ) def test_encrypted_submissions(self): """ Test encrypted submissions. """ - self._publish_xls_file(os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'transportation_encrypted.xlsx' - )) - xform = XForm.objects.get(id_string='transportation_encrypted') + self._publish_xls_file( + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "transportation_encrypted.xlsx", + ) + ) + xform = XForm.objects.get(id_string="transportation_encrypted") self.assertTrue(xform.encrypted) uuid = "c15252fe-b6f3-4853-8f04-bf89dc73985a" with self.assertRaises(Instance.DoesNotExist): Instance.objects.get(uuid=uuid) - message = u"Successful submission." + message = "Successful submission." files = {} - for filename in ['submission.xml', 'submission.xml.enc']: + for filename in ["submission.xml", "submission.xml.enc"]: files[filename] = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'instances_encrypted', filename) + self.this_directory, + "fixtures", + "transportation", + "instances_encrypted", + filename, + ) count = Instance.objects.count() acount = Attachment.objects.count() - with open(files['submission.xml.enc'], 'rb') as encryped_file: - with open(files['submission.xml'], 'rb') as f: + with open(files["submission.xml.enc"], "rb") as encryped_file: + with open(files["submission.xml"], "rb") as f: post_data = { - 'xml_submission_file': f, - 'submission.xml.enc': encryped_file} + "xml_submission_file": f, + "submission.xml.enc": encryped_file, + } self.factory = APIRequestFactory() request = self.factory.post(self._submission_url, post_data) - request.user = authenticate(username='bob', - password='bob') + request.user = authenticate(username="bob", password="bob") response = submission(request, username=self.user.username) self.assertContains(response, message, status_code=201) self.assertEqual(Instance.objects.count(), count + 1) @@ -75,7 +84,7 @@ def test_encrypted_multiple_files(self): # publish our form which contains some some repeats xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial_encrypted/tutorial_encrypted.xlsx" + "../fixtures/tutorial_encrypted/tutorial_encrypted.xlsx", ) count = XForm.objects.count() self._publish_xls_file_and_set_xform(xls_file_path) @@ -84,36 +93,36 @@ def test_encrypted_multiple_files(self): # submit an instance xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial_encrypted/instances/tutorial_encrypted.xml" + "../fixtures/tutorial_encrypted/instances/tutorial_encrypted.xml", ) encrypted_xml_submission = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial_encrypted/instances/submission.xml.enc" + "../fixtures/tutorial_encrypted/instances/submission.xml.enc", + ) + self._make_submission_w_attachment( + xml_submission_file_path, encrypted_xml_submission + ) + self.assertNotContains( + self.response, "Multiple nodes with the same name", status_code=201 ) - self._make_submission_w_attachment(xml_submission_file_path, - encrypted_xml_submission) - self.assertNotContains(self.response, - "Multiple nodes with the same name", - status_code=201) # load xml file to parse and compare # expected_list = [{u'file': u'1483528430996.jpg.enc'}, # {u'file': u'1483528445767.jpg.enc'}] - instance = Instance.objects.filter().order_by('id').last() + instance = Instance.objects.filter().order_by("id").last() self.assertEqual(instance.total_media, 3) self.assertEqual(instance.media_count, 1) self.assertFalse(instance.media_all_received) media_file_1 = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial_encrypted/instances/1483528430996.jpg.enc" + "../fixtures/tutorial_encrypted/instances/1483528430996.jpg.enc", + ) + self._make_submission_w_attachment(xml_submission_file_path, media_file_1) + self.assertNotContains( + self.response, "Multiple nodes with the same name", status_code=202 ) - self._make_submission_w_attachment(xml_submission_file_path, - media_file_1) - self.assertNotContains(self.response, - "Multiple nodes with the same name", - status_code=202) instance.refresh_from_db() self.assertEqual(instance.total_media, 3) self.assertEqual(instance.media_count, 2) @@ -121,13 +130,12 @@ def test_encrypted_multiple_files(self): media_file_2 = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial_encrypted/instances/1483528445767.jpg.enc" + "../fixtures/tutorial_encrypted/instances/1483528445767.jpg.enc", + ) + self._make_submission_w_attachment(xml_submission_file_path, media_file_2) + self.assertNotContains( + self.response, "Multiple nodes with the same name", status_code=202 ) - self._make_submission_w_attachment(xml_submission_file_path, - media_file_2) - self.assertNotContains(self.response, - "Multiple nodes with the same name", - status_code=202) instance.refresh_from_db() self.assertEqual(instance.total_media, 3) self.assertEqual(instance.media_count, 3) diff --git a/onadata/apps/logger/tests/test_form_list.py b/onadata/apps/logger/tests/test_form_list.py index 4009f196bd..a02aa03c8e 100644 --- a/onadata/apps/logger/tests/test_form_list.py +++ b/onadata/apps/logger/tests/test_form_list.py @@ -13,8 +13,8 @@ def setUp(self): def test_returns_200_for_owner(self): self._set_require_auth() - request = self.factory.get('/') - auth = DigestAuth('bob', 'bob') + request = self.factory.get("/") + auth = DigestAuth("bob", "bob") response = formList(request, self.user.username) request.META.update(auth(request.META, response)) response = formList(request, self.user.username) @@ -22,23 +22,26 @@ def test_returns_200_for_owner(self): def test_return_401_for_anon_when_require_auth_true(self): self._set_require_auth() - request = self.factory.get('/') + request = self.factory.get("/") response = formList(request, self.user.username) self.assertEqual(response.status_code, 401) def test_returns_200_for_authenticated_non_owner(self): self._set_require_auth() - credentials = ('alice', 'alice',) + credentials = ( + "alice", + "alice", + ) self._create_user(*credentials) - auth = DigestAuth('alice', 'alice') - request = self.factory.get('/') + auth = DigestAuth("alice", "alice") + request = self.factory.get("/") response = formList(request, self.user.username) request.META.update(auth(request.META, response)) response = formList(request, self.user.username) self.assertEqual(response.status_code, 200) def test_show_for_anon_when_require_auth_false(self): - request = self.factory.get('/') + request = self.factory.get("/") request.user = AnonymousUser() response = formList(request, self.user.username) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) diff --git a/onadata/apps/logger/tests/test_form_submission.py b/onadata/apps/logger/tests/test_form_submission.py index 9d92014751..52a7ee33fa 100644 --- a/onadata/apps/logger/tests/test_form_submission.py +++ b/onadata/apps/logger/tests/test_form_submission.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Test data submissions. +""" import os import re @@ -31,6 +35,7 @@ def catch_signal(signal): signal.disconnect(handler) +# pylint: disable=too-many-public-methods class TestFormSubmission(TestBase): """ Testing POSTs to "/submission" @@ -40,7 +45,7 @@ def setUp(self): TestBase.setUp(self) xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/tutorial.xlsx" + "../fixtures/tutorial/tutorial.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) @@ -50,7 +55,7 @@ def test_form_post(self): """ xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) self._make_submission(xml_submission_file_path) @@ -60,9 +65,9 @@ def test_duplicate_form_id(self): """ Should return an error if submitting to a form with a duplicate ID. """ - project = Project.objects.create(name="another project", - organization=self.user, - created_by=self.user) + project = Project.objects.create( + name="another project", organization=self.user, created_by=self.user + ) first_xform = XForm.objects.first() first_xform.pk = None first_xform.project = project @@ -70,46 +75,45 @@ def test_duplicate_form_id(self): xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 400) self.assertTrue( "Unable to submit because there are multiple forms with this form" - in self.response.content.decode('utf-8')) + in self.response.content.decode("utf-8") + ) - @patch('django.utils.datastructures.MultiValueDict.pop') + @patch("django.utils.datastructures.MultiValueDict.pop") def test_fail_with_ioerror_read(self, mock_pop): - mock_pop.side_effect = IOError( - 'request data read error') + mock_pop.side_effect = IOError("request data read error") - self.assertEquals(0, self.xform.instances.count()) + self.assertEqual(0, self.xform.instances.count()) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 400) - self.assertEquals(0, self.xform.instances.count()) + self.assertEqual(0, self.xform.instances.count()) - @patch('django.utils.datastructures.MultiValueDict.pop') + @patch("django.utils.datastructures.MultiValueDict.pop") def test_fail_with_ioerror_wsgi(self, mock_pop): - mock_pop.side_effect = IOError( - 'error during read(65536) on wsgi.input') + mock_pop.side_effect = IOError("error during read(65536) on wsgi.input") - self.assertEquals(0, self.xform.instances.count()) + self.assertEqual(0, self.xform.instances.count()) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 400) - self.assertEquals(0, self.xform.instances.count()) + self.assertEqual(0, self.xform.instances.count()) def test_submission_to_require_auth_anon(self): """ @@ -122,15 +126,16 @@ def test_submission_to_require_auth_anon(self): xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) # create a new user - username = 'alice' + username = "alice" self._create_user(username, username) - self._make_submission(xml_submission_file_path, - auth=DigestAuth('alice', 'alice')) + self._make_submission( + xml_submission_file_path, auth=DigestAuth("alice", "alice") + ) self.assertEqual(self.response.status_code, 403) def test_submission_to_require_auth_without_perm(self): @@ -144,15 +149,16 @@ def test_submission_to_require_auth_without_perm(self): xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) # create a new user - username = 'alice' + username = "alice" self._create_user(username, username) - self._make_submission(xml_submission_file_path, - auth=DigestAuth('alice', 'alice')) + self._make_submission( + xml_submission_file_path, auth=DigestAuth("alice", "alice") + ) self.assertEqual(self.response.status_code, 403) @@ -172,16 +178,16 @@ def test_submission_to_require_auth_with_perm(self): self.assertTrue(self.xform.require_auth) # create a new user - username = 'alice' + username = "alice" alice = self._create_user(username, username) # assign report perms to user - assign_perm('report_xform', alice, self.xform) + assign_perm("report_xform", alice, self.xform) auth = DigestAuth(username, username) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 201) @@ -190,7 +196,7 @@ def test_form_post_to_missing_form(self): xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "../fixtures/tutorial/instances/" - "tutorial_invalid_id_string_2012-06-27_11-27-53.xml" + "tutorial_invalid_id_string_2012-06-27_11-27-53.xml", ) with self.assertRaises(Http404): self._make_submission(path=xml_submission_file_path) @@ -201,13 +207,13 @@ def test_duplicate_submissions(self): """ xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/test_forms/survey_names/survey_names.xlsx" + "../fixtures/test_forms/survey_names/survey_names.xlsx", ) self._publish_xls_file(xls_file_path) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "../fixtures/test_forms/survey_names/instances/" - "survey_names_2012-08-17_11-24-53.xml" + "survey_names_2012-08-17_11-24-53.xml", ) self._make_submission(xml_submission_file_path) @@ -216,34 +222,38 @@ def test_duplicate_submissions(self): self.assertEqual(self.response.status_code, 202) def test_unicode_submission(self): - """Test xml submissions that contain unicode characters - """ + """Test xml submissions that contain unicode characters""" xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_unicode_submission.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_unicode_submission.xml", ) self.user.profile.require_auth = True self.user.profile.save() # create a new user - alice = self._create_user('alice', 'alice') + alice = self._create_user("alice", "alice") # assign report perms to user - assign_perm('report_xform', alice, self.xform) + assign_perm("report_xform", alice, self.xform) client = DigestClient() - client.set_authorization('alice', 'alice', 'Digest') + client.set_authorization("alice", "alice", "Digest") self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) def test_duplicate_submission_with_same_instanceid(self): - """Test duplicate xml submissions - """ + """Test duplicate xml submissions""" xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) self._make_submission(xml_submission_file_path) @@ -252,17 +262,22 @@ def test_duplicate_submission_with_same_instanceid(self): self.assertEqual(self.response.status_code, 202) def test_duplicate_submission_with_different_content(self): - """Test xml submissions with same instancID but different content - """ + """Test xml submissions with same instancID but different content""" xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) duplicate_xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid_same_instanceID.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_same_instanceID.xml", ) pre_count = Instance.objects.count() @@ -289,20 +304,18 @@ def test_edited_submission(self): xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) num_instances_history = InstanceHistory.objects.count() num_instances = Instance.objects.count() - query_args = { - 'xform': self.xform, - 'query': '{}', - 'fields': '[]', - 'count': True - } + query_args = {"xform": self.xform, "query": "{}", "fields": "[]", "count": True} cursor = [r for r in query_data(**query_args)] - num_data_instances = cursor[0]['count'] + num_data_instances = cursor[0]["count"] # make first submission self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) @@ -315,82 +328,86 @@ def test_edited_submission(self): self.assertIsNone(initial_instance.json.get(LAST_EDITED)) # no new record in instances history - self.assertEqual( - InstanceHistory.objects.count(), num_instances_history) + self.assertEqual(InstanceHistory.objects.count(), num_instances_history) # check count of mongo instances after first submission cursor = query_data(**query_args) - self.assertEqual(cursor[0]['count'], num_data_instances + 1) + self.assertEqual(cursor[0]["count"], num_data_instances + 1) # edited submission xml_edit_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml", ) client = DigestClient() - client.set_authorization('bob', 'bob', 'Digest') + client.set_authorization("bob", "bob", "Digest") with catch_signal(process_submission) as handler: self._make_submission(xml_edit_submission_file_path, client=client) self.assertEqual(self.response.status_code, 201) # we must have the same number of instances self.assertEqual(Instance.objects.count(), num_instances + 1) # should be a new record in instances history - self.assertEqual( - InstanceHistory.objects.count(), num_instances_history + 1) + self.assertEqual(InstanceHistory.objects.count(), num_instances_history + 1) instance_history_1 = InstanceHistory.objects.first() edited_instance = self.xform.instances.first() - self.assertDictEqual(initial_instance.get_dict(), - instance_history_1.get_dict()) - handler.assert_called_once_with(instance=edited_instance, - sender=Instance, signal=ANY) + self.assertDictEqual(initial_instance.get_dict(), instance_history_1.get_dict()) + handler.assert_called_once_with( + instance=edited_instance, sender=Instance, signal=ANY + ) self.assertNotEqual(edited_instance.uuid, instance_history_1.uuid) # check that instance history's submission_date is equal to instance's # date_created - last_edited by default is null for an instance - self.assertEquals(edited_instance.date_created, - instance_history_1.submission_date) + self.assertEqual( + edited_instance.date_created, instance_history_1.submission_date + ) # check that '_last_edited' key is not in the json self.assertIn(LAST_EDITED, edited_instance.json) cursor = query_data(**query_args) - self.assertEqual(cursor[0]['count'], num_data_instances + 1) + self.assertEqual(cursor[0]["count"], num_data_instances + 1) # make sure we edited the mongo db record and NOT added a new row - query_args['count'] = False + query_args["count"] = False cursor = query_data(**query_args) record = cursor[0] with open(xml_edit_submission_file_path, "r") as f: xml_str = f.read() xml_str = clean_and_parse_xml(xml_str).toxml() edited_name = re.match(r"^.+?(.+?)", xml_str).groups()[0] - self.assertEqual(record['name'], edited_name) + self.assertEqual(record["name"], edited_name) instance_before_second_edit = edited_instance xml_edit_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid_edited_again.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited_again.xml", ) self._make_submission(xml_edit_submission_file_path) cursor = query_data(**query_args) record = cursor[0] edited_instance = self.xform.instances.first() instance_history_2 = InstanceHistory.objects.last() - self.assertEquals(instance_before_second_edit.last_edited, - instance_history_2.submission_date) + self.assertEqual( + instance_before_second_edit.last_edited, instance_history_2.submission_date + ) # check that '_last_edited' key is not in the json self.assertIn(LAST_EDITED, edited_instance.json) - self.assertEqual(record['name'], 'Tom and Jerry') - self.assertEqual( - InstanceHistory.objects.count(), num_instances_history + 2) + self.assertEqual(record["name"], "Tom and Jerry") + self.assertEqual(InstanceHistory.objects.count(), num_instances_history + 2) # submitting original submission is treated as a duplicate # does not add a new record # does not change data self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 202) self.assertEqual(Instance.objects.count(), num_instances + 1) - self.assertEqual( - InstanceHistory.objects.count(), num_instances_history + 2) + self.assertEqual(InstanceHistory.objects.count(), num_instances_history + 2) def test_submission_w_mismatched_uuid(self): """ @@ -400,8 +417,11 @@ def test_submission_w_mismatched_uuid(self): # submit instance with uuid that would not match the forms xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_xform_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_xform_uuid.xml", ) self._make_submission(xml_submission_file_path) @@ -414,47 +434,53 @@ def _test_fail_submission_if_no_username(self): """ xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_xform_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_xform_uuid.xml", ) with self.assertRaises(Http404): - self._make_submission(path=xml_submission_file_path, username='') + self._make_submission(path=xml_submission_file_path, username="") def test_fail_submission_if_bad_id_string(self): - """Test that a submission fails if the uuid's don't match. - """ + """Test that a submission fails if the uuid's don't match.""" xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_bad_id_string.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_bad_id_string.xml", ) with self.assertRaises(Http404): self._make_submission(path=xml_submission_file_path) def test_edit_updated_geopoint_cache(self): - query_args = { - 'xform': self.xform, - 'query': '{}', - 'fields': '[]', - 'count': True - } + query_args = {"xform": self.xform, "query": "{}", "fields": "[]", "count": True} xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) # query mongo for the _geopoint field - query_args['count'] = False + query_args["count"] = False records = query_data(**query_args) self.assertEqual(len(records), 1) # submit the edited instance xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml", ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) @@ -462,7 +488,7 @@ def test_edit_updated_geopoint_cache(self): self.assertEqual(len(records), 1) cached_geopoint = records[0][GEOLOCATION] # the cached geopoint should equal the gps field - gps = records[0]['gps'].split(" ") + gps = records[0]["gps"].split(" ") self.assertEqual(float(gps[0]), float(cached_geopoint[0])) self.assertEqual(float(gps[1]), float(cached_geopoint[1])) @@ -471,18 +497,17 @@ def test_submission_when_requires_auth(self): self.user.profile.save() # create a new user - alice = self._create_user('alice', 'alice') + alice = self._create_user("alice", "alice") # assign report perms to user - assign_perm('report_xform', alice, self.xform) + assign_perm("report_xform", alice, self.xform) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) - auth = DigestAuth('alice', 'alice') - self._make_submission( - xml_submission_file_path, auth=auth) + auth = DigestAuth("alice", "alice") + self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 201) def test_submission_linked_to_reporter(self): @@ -490,19 +515,18 @@ def test_submission_linked_to_reporter(self): self.user.profile.save() # create a new user - alice = self._create_user('alice', 'alice') + alice = self._create_user("alice", "alice") UserProfile.objects.create(user=alice) # assign report perms to user - assign_perm('report_xform', alice, self.xform) + assign_perm("report_xform", alice, self.xform) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) - auth = DigestAuth('alice', 'alice') - self._make_submission( - xml_submission_file_path, auth=auth) + auth = DigestAuth("alice", "alice") + self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 201) instance = Instance.objects.all().reverse()[0] self.assertEqual(instance.user, alice) @@ -513,8 +537,11 @@ def test_edited_submission_require_auth(self): """ xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) # require authentication self.user.profile.require_auth = True @@ -522,80 +549,76 @@ def test_edited_submission_require_auth(self): num_instances_history = InstanceHistory.objects.count() num_instances = Instance.objects.count() - query_args = { - 'xform': self.xform, - 'query': '{}', - 'fields': '[]', - 'count': True - } + query_args = {"xform": self.xform, "query": "{}", "fields": "[]", "count": True} cursor = query_data(**query_args) - num_data_instances = cursor[0]['count'] + num_data_instances = cursor[0]["count"] # make first submission self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) self.assertEqual(Instance.objects.count(), num_instances + 1) # no new record in instances history - self.assertEqual( - InstanceHistory.objects.count(), num_instances_history) + self.assertEqual(InstanceHistory.objects.count(), num_instances_history) # check count of mongo instances after first submission cursor = query_data(**query_args) - self.assertEqual(cursor[0]['count'], num_data_instances + 1) + self.assertEqual(cursor[0]["count"], num_data_instances + 1) # create a new user - alice = self._create_user('alice', 'alice') + alice = self._create_user("alice", "alice") UserProfile.objects.create(user=alice) - auth = DigestAuth('alice', 'alice') + auth = DigestAuth("alice", "alice") # edited submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml", ) self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 403) # assign report perms to user - assign_perm('report_xform', alice, self.xform) - assign_perm('logger.change_xform', alice, self.xform) + assign_perm("report_xform", alice, self.xform) + assign_perm("logger.change_xform", alice, self.xform) self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 201) # we must have the same number of instances self.assertEqual(Instance.objects.count(), num_instances + 1) # should be a new record in instances history - self.assertEqual( - InstanceHistory.objects.count(), num_instances_history + 1) + self.assertEqual(InstanceHistory.objects.count(), num_instances_history + 1) cursor = query_data(**query_args) - self.assertEqual(cursor[0]['count'], num_data_instances + 1) + self.assertEqual(cursor[0]["count"], num_data_instances + 1) # make sure we edited the mongo db record and NOT added a new row - query_args['count'] = False + query_args["count"] = False cursor = query_data(**query_args) record = cursor[0] with open(xml_submission_file_path, "r") as f: xml_str = f.read() xml_str = clean_and_parse_xml(xml_str).toxml() edited_name = re.match(r"^.+?(.+?)", xml_str).groups()[0] - self.assertEqual(record['name'], edited_name) + self.assertEqual(record["name"], edited_name) - @patch('onadata.libs.utils.logger_tools.create_instance') + @patch("onadata.libs.utils.logger_tools.create_instance") def test_fail_with_unreadable_post_error(self, mock_create_instance): """Test UnreadablePostError is handled on form data submission""" mock_create_instance.side_effect = UnreadablePostError( - 'error during read(65536) on wsgi.input' + "error during read(65536) on wsgi.input" ) - self.assertEquals(0, self.xform.instances.count()) + self.assertEqual(0, self.xform.instances.count()) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 400) - self.assertEquals(0, self.xform.instances.count()) + self.assertEqual(0, self.xform.instances.count()) def test_form_submission_with_infinity_values(self): """ @@ -607,15 +630,15 @@ def test_form_submission_with_infinity_values(self): This test confirms that we are handling such cases and they do not result in 500 response codes. """ - xls_file_path = os.path.join(os.path.dirname( - os.path.abspath(__file__)), "../tests/fixtures/infinity.xlsx" + xls_file_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "../tests/fixtures/infinity.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) - xml_submission_file_path = os.path.join(os.path.dirname( - os.path.abspath(__file__)), "../tests/fixtures/infinity.xml" + xml_submission_file_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "../tests/fixtures/infinity.xml" ) self._make_submission(path=xml_submission_file_path) - self.assertEquals(400, self.response.status_code) - self.assertIn( - 'invalid input syntax for type json', str(self.response.message)) + self.assertEqual(400, self.response.status_code) + self.assertIn("invalid input syntax for type json", str(self.response.message)) diff --git a/onadata/apps/logger/tests/test_importing_database.py b/onadata/apps/logger/tests/test_importing_database.py index cdb185ef9f..bc671560d9 100644 --- a/onadata/apps/logger/tests/test_importing_database.py +++ b/onadata/apps/logger/tests/test_importing_database.py @@ -11,31 +11,39 @@ CUR_PATH = os.path.abspath(__file__) CUR_DIR = os.path.dirname(CUR_PATH) -DB_FIXTURES_PATH = os.path.join(CUR_DIR, 'data_from_sdcard') +DB_FIXTURES_PATH = os.path.join(CUR_DIR, "data_from_sdcard") def images_count(username="bob"): images = glob.glob( - os.path.join(settings.MEDIA_ROOT, username, 'attachments', '*', '*')) + os.path.join(settings.MEDIA_ROOT, username, "attachments", "*", "*") + ) return len(images) class TestImportingDatabase(TestBase): - def setUp(self): TestBase.setUp(self) self._publish_xls_file( os.path.join( - settings.PROJECT_ROOT, "apps", "logger", "fixtures", - "test_forms", "tutorial.xlsx")) + settings.PROJECT_ROOT, + "apps", + "logger", + "fixtures", + "test_forms", + "tutorial.xlsx", + ) + ) def tearDown(self): # delete everything we imported Instance.objects.all().delete() # ? if settings.TESTING_MODE: images = glob.glob( - os.path.join(settings.MEDIA_ROOT, self.user.username, - 'attachments', '*', '*')) + os.path.join( + settings.MEDIA_ROOT, self.user.username, "attachments", "*", "*" + ) + ) for image in images: os.remove(image) @@ -56,8 +64,9 @@ def test_importing_b1_and_b2(self): initial_instance_count = Instance.objects.count() initial_image_count = images_count() - import_instances_from_zip(os.path.join( - DB_FIXTURES_PATH, "bulk_submission.zip"), self.user) + import_instances_from_zip( + os.path.join(DB_FIXTURES_PATH, "bulk_submission.zip"), self.user + ) instance_count = Instance.objects.count() image_count = images_count() @@ -71,31 +80,29 @@ def test_importing_b1_and_b2(self): def test_badzipfile_import(self): total, success, errors = import_instances_from_zip( - os.path.join( - CUR_DIR, "Water_Translated_2011_03_10.xml"), self.user) + os.path.join(CUR_DIR, "Water_Translated_2011_03_10.xml"), self.user + ) self.assertEqual(total, 0) self.assertEqual(success, 0) - expected_errors = [u'File is not a zip file'] + expected_errors = ["File is not a zip file"] self.assertEqual(errors, expected_errors) def test_bulk_import_post(self): zip_file_path = os.path.join( - DB_FIXTURES_PATH, "bulk_submission_w_extra_instance.zip") - url = reverse(bulksubmission, kwargs={ - "username": self.user.username - }) + DB_FIXTURES_PATH, "bulk_submission_w_extra_instance.zip" + ) + url = reverse(bulksubmission, kwargs={"username": self.user.username}) with open(zip_file_path, "rb") as zip_file: - post_data = {'zip_submission_file': zip_file} + post_data = {"zip_submission_file": zip_file} response = self.client.post(url, post_data) self.assertEqual(response.status_code, 200) def test_bulk_import_post_with_username_in_uppercase(self): zip_file_path = os.path.join( - DB_FIXTURES_PATH, "bulk_submission_w_extra_instance.zip") - url = reverse(bulksubmission, kwargs={ - "username": self.user.username.upper() - }) + DB_FIXTURES_PATH, "bulk_submission_w_extra_instance.zip" + ) + url = reverse(bulksubmission, kwargs={"username": self.user.username.upper()}) with open(zip_file_path, "rb") as zip_file: - post_data = {'zip_submission_file': zip_file} + post_data = {"zip_submission_file": zip_file} response = self.client.post(url, post_data) self.assertEqual(response.status_code, 200) diff --git a/onadata/apps/logger/tests/test_instance_creation.py b/onadata/apps/logger/tests/test_instance_creation.py index 574cf814cc..9d8535672e 100644 --- a/onadata/apps/logger/tests/test_instance_creation.py +++ b/onadata/apps/logger/tests/test_instance_creation.py @@ -37,12 +37,10 @@ def create_post_data(path): def get_absolute_path(subdirectory): - return os.path.join( - os.path.dirname(os.path.abspath(__file__)), subdirectory) + return os.path.join(os.path.dirname(os.path.abspath(__file__)), subdirectory) class TestInstanceCreation(TestCase): - def setUp(self): self.user = User.objects.create(username="bob") profile, c = UserProfile.objects.get_or_create(user=self.user) @@ -52,34 +50,46 @@ def setUp(self): absolute_path = get_absolute_path("forms") open_forms = open_all_files(absolute_path) - self.json = '{"default_language": "default", ' \ - '"id_string": "Water_2011_03_17", "children": [], ' \ - '"name": "Water_2011_03_17", ' \ - '"title": "Water_2011_03_17", "type": "survey"}' + self.json = ( + '{"default_language": "default", ' + '"id_string": "Water_2011_03_17", "children": [], ' + '"name": "Water_2011_03_17", ' + '"title": "Water_2011_03_17", "type": "survey"}' + ) for path, open_file in open_forms.items(): XForm.objects.create( - xml=open_file.read(), user=self.user, json=self.json, - require_auth=False, project=self.project) + xml=open_file.read(), + user=self.user, + json=self.json, + require_auth=False, + project=self.project, + ) open_file.close() self._create_water_translated_form() def _create_water_translated_form(self): - f = open(os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "Water_Translated_2011_03_10.xml" - )) + f = open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "Water_Translated_2011_03_10.xml", + ) + ) xml = f.read() f.close() self.xform = XForm.objects.create( - xml=xml, user=self.user, json=self.json, project=self.project) + xml=xml, user=self.user, json=self.json, project=self.project + ) def test_form_submission(self): # no more submission to non-existent form, # setUp ensures the Water_Translated_2011_03_10 xform is valid - f = open(os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "Water_Translated_2011_03_10_2011-03-10_14-38-28.xml")) + f = open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "Water_Translated_2011_03_10_2011-03-10_14-38-28.xml", + ) + ) xml = f.read() f.close() Instance.objects.create(xml=xml, user=self.user, xform=self.xform) @@ -89,14 +99,16 @@ def test_data_submission(self): for subdirectory in subdirectories: path = get_absolute_path(subdirectory) postdata = create_post_data(path) - response = self.client.post('/bob/submission', postdata) + response = self.client.post("/bob/submission", postdata) self.failUnlessEqual(response.status_code, 201) def test_submission_for_missing_form(self): - xml_file = open(os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "Health_2011_03_13_invalid_id_string.xml" - )) + xml_file = open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "Health_2011_03_13_invalid_id_string.xml", + ) + ) postdata = {"xml_submission_file": xml_file} - response = self.client.post('/bob/submission', postdata) + response = self.client.post("/bob/submission", postdata) self.failUnlessEqual(response.status_code, 404) diff --git a/onadata/apps/logger/tests/test_parsing.py b/onadata/apps/logger/tests/test_parsing.py index 60928091a4..48075fb213 100644 --- a/onadata/apps/logger/tests/test_parsing.py +++ b/onadata/apps/logger/tests/test_parsing.py @@ -5,19 +5,26 @@ from xml.dom import minidom from onadata.apps.main.tests.test_base import TestBase -from onadata.apps.logger.xform_instance_parser import XFormInstanceParser,\ - xpath_from_xml_node -from onadata.apps.logger.xform_instance_parser import get_uuid_from_xml,\ - get_meta_from_xml, get_deprecated_uuid_from_xml +from onadata.apps.logger.xform_instance_parser import ( + XFormInstanceParser, + xpath_from_xml_node, +) +from onadata.apps.logger.xform_instance_parser import ( + get_uuid_from_xml, + get_meta_from_xml, + get_deprecated_uuid_from_xml, +) from onadata.libs.utils.common_tags import XFORM_ID_STRING from onadata.apps.logger.models.xform import XForm -from onadata.apps.logger.xform_instance_parser import _xml_node_to_dict,\ - clean_and_parse_xml +from onadata.apps.logger.xform_instance_parser import ( + _xml_node_to_dict, + clean_and_parse_xml, +) -XML = u"xml" -DICT = u"dict" -FLAT_DICT = u"flat_dict" +XML = "xml" +DICT = "dict" +FLAT_DICT = "flat_dict" ID = XFORM_ID_STRING @@ -27,7 +34,7 @@ def _publish_and_submit_new_repeats(self): # publish our form which contains some some repeats xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/new_repeats/new_repeats.xlsx" + "../fixtures/new_repeats/new_repeats.xlsx", ) count = XForm.objects.count() self._publish_xls_file_and_set_xform(xls_file_path) @@ -36,8 +43,7 @@ def _publish_and_submit_new_repeats(self): # submit an instance xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/new_repeats/instances/" - "new_repeats_2012-07-05-14-33-53.xml" + "../fixtures/new_repeats/instances/" "new_repeats_2012-07-05-14-33-53.xml", ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) @@ -52,85 +58,88 @@ def test_parse_xform_nested_repeats(self): parser = XFormInstanceParser(self.xml, self.xform) dict = parser.to_dict() expected_dict = { - u'new_repeats': { - u'info': - { - u'age': u'80', - u'name': u'Adam' - }, - u'kids': - { - u'kids_details': - [ - { - u'kids_age': u'50', - u'kids_name': u'Abel' - }, + "new_repeats": { + "info": {"age": "80", "name": "Adam"}, + "kids": { + "kids_details": [ + {"kids_age": "50", "kids_name": "Abel"}, ], - u'has_kids': u'1' + "has_kids": "1", }, - u'web_browsers': u'chrome ie', - u'gps': u'-1.2627557 36.7926442 0.0 30.0' + "web_browsers": "chrome ie", + "gps": "-1.2627557 36.7926442 0.0 30.0", } } self.assertEqual(dict, expected_dict) flat_dict = parser.to_flat_dict() expected_flat_dict = { - u'gps': u'-1.2627557 36.7926442 0.0 30.0', - u'kids/kids_details': - [ + "gps": "-1.2627557 36.7926442 0.0 30.0", + "kids/kids_details": [ { - u'kids/kids_details/kids_name': u'Abel', - u'kids/kids_details/kids_age': u'50' + "kids/kids_details/kids_name": "Abel", + "kids/kids_details/kids_age": "50", } ], - u'kids/has_kids': u'1', - u'info/age': u'80', - u'web_browsers': u'chrome ie', - u'info/name': u'Adam' + "kids/has_kids": "1", + "info/age": "80", + "web_browsers": "chrome ie", + "info/name": "Adam", } self.assertEqual(flat_dict, expected_flat_dict) def test_xpath_from_xml_node(self): - xml_str = '' \ - 'c911d71ce1ac48478e5f8bac99addc4e' \ - '-1.2625149 36.7924478 0.0 30.0' \ - 'Yo' \ - '-1.2625072 36.7924328 0.0 30.0' \ - 'What' + xml_str = ( + "' + "c911d71ce1ac48478e5f8bac99addc4e" + "-1.2625149 36.7924478 0.0 30.0" + "Yo" + "-1.2625072 36.7924328 0.0 30.0" + "What" + ) clean_xml_str = xml_str.strip() - clean_xml_str = re.sub(r">\s+<", u"><", clean_xml_str) + clean_xml_str = re.sub(r">\s+<", "><", clean_xml_str) root_node = minidom.parseString(clean_xml_str).documentElement # get the first top-level gps element gps_node = root_node.firstChild.nextSibling - self.assertEqual(gps_node.nodeName, u'gps') + self.assertEqual(gps_node.nodeName, "gps") # get the info element within the gps element - info_node = gps_node.getElementsByTagName(u'info')[0] + info_node = gps_node.getElementsByTagName("info")[0] # create an xpath that should look like gps/info xpath = xpath_from_xml_node(info_node) - self.assertEqual(xpath, u'gps/info') + self.assertEqual(xpath, "gps/info") def test_get_meta_from_xml(self): with open( os.path.join( - os.path.dirname(__file__), "..", "fixtures", "tutorial", - "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml"), - "r") as xml_file: + os.path.dirname(__file__), + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml", + ), + "r", + ) as xml_file: xml_str = xml_file.read() instanceID = get_meta_from_xml(xml_str, "instanceID") - self.assertEqual(instanceID, - "uuid:2d8c59eb-94e9-485d-a679-b28ffe2e9b98") + self.assertEqual(instanceID, "uuid:2d8c59eb-94e9-485d-a679-b28ffe2e9b98") deprecatedID = get_meta_from_xml(xml_str, "deprecatedID") self.assertEqual(deprecatedID, "uuid:729f173c688e482486a48661700455ff") def test_get_meta_from_xml_without_uuid_returns_none(self): with open( os.path.join( - os.path.dirname(__file__), "..", "fixtures", "tutorial", - "instances", "tutorial_2012-06-27_11-27-53.xml"), - "r") as xml_file: + os.path.dirname(__file__), + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53.xml", + ), + "r", + ) as xml_file: xml_str = xml_file.read() instanceID = get_meta_from_xml(xml_str, "instanceID") self.assertIsNone(instanceID) @@ -138,9 +147,15 @@ def test_get_meta_from_xml_without_uuid_returns_none(self): def test_get_uuid_from_xml(self): with open( os.path.join( - os.path.dirname(__file__), "..", "fixtures", "tutorial", - "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml"), - "r") as xml_file: + os.path.dirname(__file__), + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", + ), + "r", + ) as xml_file: xml_str = xml_file.read() instanceID = get_uuid_from_xml(xml_str) self.assertEqual(instanceID, "729f173c688e482486a48661700455ff") @@ -148,9 +163,15 @@ def test_get_uuid_from_xml(self): def test_get_deprecated_uuid_from_xml(self): with open( os.path.join( - os.path.dirname(__file__), "..", "fixtures", "tutorial", - "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml"), - "r") as xml_file: + os.path.dirname(__file__), + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml", + ), + "r", + ) as xml_file: xml_str = xml_file.read() deprecatedID = get_deprecated_uuid_from_xml(xml_str) self.assertEqual(deprecatedID, "729f173c688e482486a48661700455ff") @@ -160,7 +181,7 @@ def test_parse_xform_nested_repeats_multiple_nodes(self): # publish our form which contains some some repeats xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/new_repeats/new_repeats.xlsx" + "../fixtures/new_repeats/new_repeats.xlsx", ) count = XForm.objects.count() self._publish_xls_file_and_set_xform(xls_file_path) @@ -169,18 +190,17 @@ def test_parse_xform_nested_repeats_multiple_nodes(self): # submit an instance xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/new_repeats/instances/" - "multiple_nodes_error.xml" + "../fixtures/new_repeats/instances/" "multiple_nodes_error.xml", ) self._make_submission(xml_submission_file_path) - self.assertEquals(201, self.response.status_code) + self.assertEqual(201, self.response.status_code) def test_multiple_media_files_on_encrypted_form(self): self._create_user_and_login() # publish our form which contains some some repeats xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial_encrypted/tutorial_encrypted.xlsx" + "../fixtures/tutorial_encrypted/tutorial_encrypted.xlsx", ) count = XForm.objects.count() self._publish_xls_file_and_set_xform(xls_file_path) @@ -189,12 +209,12 @@ def test_multiple_media_files_on_encrypted_form(self): # submit an instance xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial_encrypted/instances/tutorial_encrypted.xml" + "../fixtures/tutorial_encrypted/instances/tutorial_encrypted.xml", ) self._make_submission(xml_submission_file_path) - self.assertNotContains(self.response, - "Multiple nodes with the same name", - status_code=201) + self.assertNotContains( + self.response, "Multiple nodes with the same name", status_code=201 + ) # load xml file to parse and compare xml_file = open(xml_submission_file_path) @@ -204,28 +224,30 @@ def test_multiple_media_files_on_encrypted_form(self): parser = XFormInstanceParser(self.xml, self.xform) dict = parser.to_dict() - expected_list = [{u'file': u'1483528430996.jpg.enc'}, - {u'file': u'1483528445767.jpg.enc'}] - self.assertEqual(dict.get('data').get('media'), expected_list) + expected_list = [ + {"file": "1483528430996.jpg.enc"}, + {"file": "1483528445767.jpg.enc"}, + ] + self.assertEqual(dict.get("data").get("media"), expected_list) flat_dict = parser.to_flat_dict() - expected_flat_list = [{u'media/file': u'1483528430996.jpg.enc'}, - {u'media/file': u'1483528445767.jpg.enc'}] - self.assertEqual(flat_dict.get('media'), expected_flat_list) + expected_flat_list = [ + {"media/file": "1483528430996.jpg.enc"}, + {"media/file": "1483528445767.jpg.enc"}, + ] + self.assertEqual(flat_dict.get("media"), expected_flat_list) def test_xml_repeated_nodes_to_dict(self): xml_file = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "../fixtures/repeated_nodes.xml" + os.path.dirname(os.path.abspath(__file__)), "../fixtures/repeated_nodes.xml" ) json_file = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/repeated_nodes_expected_results.json" + "../fixtures/repeated_nodes_expected_results.json", ) with open(xml_file) as file: xml_dict = _xml_node_to_dict(clean_and_parse_xml(file.read())) - self.assertTrue(xml_dict['#document']['RW_OUNIS_2016']['S2A']) - self.assertEqual(3, len( - xml_dict['#document']['RW_OUNIS_2016']['S2A'])) + self.assertTrue(xml_dict["#document"]["RW_OUNIS_2016"]["S2A"]) + self.assertEqual(3, len(xml_dict["#document"]["RW_OUNIS_2016"]["S2A"])) with open(json_file) as file: self.assertEqual(json.loads(file.read()), xml_dict) diff --git a/onadata/apps/logger/tests/test_publish_xls.py b/onadata/apps/logger/tests/test_publish_xls.py index 36ee8a39a1..10192e3087 100644 --- a/onadata/apps/logger/tests/test_publish_xls.py +++ b/onadata/apps/logger/tests/test_publish_xls.py @@ -11,13 +11,12 @@ class TestPublishXLS(TestBase): - def test_publish_xls(self): xls_file_path = os.path.join( - self.this_directory, "fixtures", - "transportation", "transportation.xlsx") + self.this_directory, "fixtures", "transportation", "transportation.xlsx" + ) count = XForm.objects.count() - call_command('publish_xls', xls_file_path, self.user.username) + call_command("publish_xls", xls_file_path, self.user.username) self.assertEqual(XForm.objects.count(), count + 1) form = XForm.objects.get() self.assertFalse(form.require_auth) @@ -25,26 +24,36 @@ def test_publish_xls(self): def test_publish_xls_replacement(self): count = XForm.objects.count() xls_file_path = os.path.join( - self.this_directory, "fixtures", - "transportation", "transportation.xlsx") - call_command('publish_xls', xls_file_path, self.user.username) + self.this_directory, "fixtures", "transportation", "transportation.xlsx" + ) + call_command("publish_xls", xls_file_path, self.user.username) self.assertEqual(XForm.objects.count(), count + 1) count = XForm.objects.count() xls_file_path = os.path.join( - self.this_directory, "fixtures", - "transportation", "transportation_updated.xlsx") + self.this_directory, + "fixtures", + "transportation", + "transportation_updated.xlsx", + ) # call command without replace param with self.assertRaises(CommandError): - call_command('publish_xls', xls_file_path, self.user.username) + call_command("publish_xls", xls_file_path, self.user.username) # now we call the command with the replace param - call_command( - 'publish_xls', xls_file_path, self.user.username, replace=True) + call_command("publish_xls", xls_file_path, self.user.username, replace=True) # count should remain the same self.assertEqual(XForm.objects.count(), count) # check if the extra field has been added - self.xform = XForm.objects.order_by('id').reverse()[0] - is_updated_form = len([e.name for e in self.xform.survey_elements - if e.name == u'preferred_means']) > 0 + self.xform = XForm.objects.order_by("id").reverse()[0] + is_updated_form = ( + len( + [ + e.name + for e in self.xform.survey_elements + if e.name == "preferred_means" + ] + ) + > 0 + ) self.assertTrue(is_updated_form) def test_xform_hash(self): @@ -59,12 +68,12 @@ def test_xform_hash(self): self.assertFalse(self.xform.hash == "" or self.xform.hash is None) self.assertEqual(self.xform.hash, self.xform.get_hash()) # test that the md5 value of the hash is as expected - calculated_hash = md5(self.xform.xml.encode('utf8')).hexdigest() + calculated_hash = md5(self.xform.xml.encode("utf8")).hexdigest() self.assertEqual(self.xform.hash[4:], calculated_hash) # assert that the hash changes when you change the form title xform_old_hash = self.xform.hash self.xform.title = "Hunter 2 Rules" - self.xform.save(update_fields=['title']) + self.xform.save(update_fields=["title"]) self.assertFalse(self.xform.hash == "" or self.xform.hash is None) self.assertFalse(self.xform.hash == xform_old_hash) @@ -75,8 +84,9 @@ def test_report_exception_with_exc_info(self): except Exception as e: exc_info = sys.exc_info() try: - report_exception(subject="Test report exception", info=e, - exc_info=exc_info) + report_exception( + subject="Test report exception", info=e, exc_info=exc_info + ) except Exception as e: raise AssertionError("%s" % e) @@ -89,10 +99,10 @@ def test_report_exception_without_exc_info(self): def test_publish_xls_version(self): xls_file_path = os.path.join( - self.this_directory, "fixtures", - "transportation", "transportation.xlsx") + self.this_directory, "fixtures", "transportation", "transportation.xlsx" + ) count = XForm.objects.count() - call_command('publish_xls', xls_file_path, self.user.username) + call_command("publish_xls", xls_file_path, self.user.username) self.assertEqual(XForm.objects.count(), count + 1) form = XForm.objects.get() self.assertIsNotNone(form.version) diff --git a/onadata/apps/logger/tests/test_simple_submission.py b/onadata/apps/logger/tests/test_simple_submission.py index 5c060e8ae0..c8fd456872 100644 --- a/onadata/apps/logger/tests/test_simple_submission.py +++ b/onadata/apps/logger/tests/test_simple_submission.py @@ -7,9 +7,7 @@ from onadata.apps.logger.xform_instance_parser import DuplicateInstance from onadata.apps.main.models import UserProfile from onadata.apps.viewer.models.data_dictionary import DataDictionary -from onadata.libs.utils.logger_tools import ( - create_instance, safe_create_instance -) +from onadata.libs.utils.logger_tools import create_instance, safe_create_instance from onadata.libs.utils.user_auth import get_user_default_project @@ -18,8 +16,9 @@ class TempFileProxy(object): create_instance will be looking for a file object, with "read" and "close" methods. """ + def __init__(self, content): - self.content = content.encode('utf-8') + self.content = content.encode("utf-8") def read(self): return self.content @@ -33,46 +32,55 @@ def _get_xml_for_form(self, xform): builder = SurveyElementBuilder() sss = builder.create_survey_element_from_json(xform.json) xform.xml = sss.to_xml() - xform._mark_start_time_boolean() + xform.mark_start_time_boolean() xform.save() def _submit_at_hour(self, hour): - st_xml = '2012-01-11T%d:00:00.000+00'\ - '' % hour + st_xml = ( + "2012-01-11T%d:00:00.000+00' + "" % hour + ) try: create_instance(self.user.username, TempFileProxy(st_xml), []) except DuplicateInstance: pass def _submit_simple_yes(self): - create_instance(self.user.username, TempFileProxy( - '' - 'Yes'), []) + create_instance( + self.user.username, + TempFileProxy( + "" + "Yes" + ), + [], + ) def setUp(self): - self.user = User.objects.create( - username="admin", email="sample@example.com") + self.user = User.objects.create(username="admin", email="sample@example.com") self.project = UserProfile.objects.create(user=self.user) self.user.set_password("pass") self.project = get_user_default_project(self.user) self.xform1 = DataDictionary() self.xform1.user = self.user self.xform1.project = self.project - self.xform1.json = '{"id_string": "yes_or_no_form", "children": '\ - '[{"name": '\ - '"yesno", "label": "Yes or no?", "type": "text"}],'\ - ' "name": "yes_or_no", "title": "yes_or_no", "type'\ - '": "survey"}'.strip() + self.xform1.json = ( + '{"id_string": "yes_or_no_form", "children": ' + '[{"name": ' + '"yesno", "label": "Yes or no?", "type": "text"}],' + ' "name": "yes_or_no", "title": "yes_or_no", "type' + '": "survey"}'.strip() + ) self.xform2 = DataDictionary() self.xform2.user = self.user self.xform2.project = self.project - self.xform2.json = '{"id_string": "start_time_form", "children": '\ - '[{"name":'\ - '"start_time", "type": "start"}], "name": "start_t'\ - 'ime_form", "title": "start_time_form",'\ - '"type": "survey"}'\ - .strip() + self.xform2.json = ( + '{"id_string": "start_time_form", "children": ' + '[{"name":' + '"start_time", "type": "start"}], "name": "start_t' + 'ime_form", "title": "start_time_form",' + '"type": "survey"}'.strip() + ) self._get_xml_for_form(self.xform1) self._get_xml_for_form(self.xform2) @@ -82,40 +90,40 @@ def test_start_time_boolean_properly_set(self): self.assertTrue(self.xform2.has_start_time) def test_simple_yes_submission(self): - self.assertEquals(0, self.xform1.instances.count()) + self.assertEqual(0, self.xform1.instances.count()) self._submit_simple_yes() - self.assertEquals(1, self.xform1.instances.count()) + self.assertEqual(1, self.xform1.instances.count()) self._submit_simple_yes() # a simple "yes" submission *SHOULD* increment the survey count - self.assertEquals(2, self.xform1.instances.count()) + self.assertEqual(2, self.xform1.instances.count()) def test_start_time_submissions(self): """This test checks to make sure that instances *with start_time available* are marked as duplicates when the XML is a direct match. """ - self.assertEquals(0, self.xform2.instances.count()) + self.assertEqual(0, self.xform2.instances.count()) self._submit_at_hour(11) - self.assertEquals(1, self.xform2.instances.count()) + self.assertEqual(1, self.xform2.instances.count()) self._submit_at_hour(12) - self.assertEquals(2, self.xform2.instances.count()) + self.assertEqual(2, self.xform2.instances.count()) # an instance from 11 AM already exists in the database, so it # *SHOULD NOT* increment the survey count. self._submit_at_hour(11) - self.assertEquals(2, self.xform2.instances.count()) + self.assertEqual(2, self.xform2.instances.count()) def test_corrupted_submission(self): - """Test xml submissions that contain unicode characters. - """ - xml = 'v\xee\xf3\xc0k\x91\x91\xae\xff\xff\xff\xff\xcf[$b\xd0\xc9\'uW\x80RP\xff\xff\xff\xff7\xd0\x03%F\xa7p\xa2\x87\xb6f\xb1\xff\xff\xff\xffg~\xf3O\xf3\x9b\xbc\xf6ej_$\xff\xff\xff\xff\x13\xe8\xa9D\xed\xfb\xe7\xa4d\x96>\xfa\xff\xff\xff\xff\xc7h"\x86\x14\\.\xdb\x8aoF\xa4\xff\xff\xff\xff\xcez\xff\x01\x0c\x9a\x94\x18\xe1\x03\x8e\xfa\xff\xff\xff\xff39P|\xf9n\x18F\xb1\xcb\xacd\xff\xff\xff\xff\xce>\x97i;1u\xcfI*\xf2\x8e\xff\xff\xff\xffFg\x9d\x0fR:\xcd*\x14\x85\xf0e\xff\xff\xff\xff\xd6\xdc\xda\x8eM\x06\xf1\xfc\xc1\xe8\xd6\xe0\xff\xff\xff\xff\xe7G\xe1\xa1l\x02T\n\xde\x1boJ\xff\xff\xff\xffz \x92\xbc\tR{#\xbb\x9f\xa6s\xff\xff\xff\xff\xa2\x8f(\xb6=\xe11\xfcV\xcf\xef\x0b\xff\xff\xff\xff\xa3\x83\x7ft\xd7\x05+)\xeb9\\*\xff\xff\xff\xff\xfe\x93\xb2\xa2\x06n;\x1b4\xaf\xa6\x93\xff\xff\xff\xff\xe7\xf7\x12Q\x83\xbb\x9a\xc8\xc8q34\xff\xff\xff\xffT2\xa5\x07\x9a\xc9\x89\xf8\x14Y\xab\x19\xff\xff\xff\xff\x16\xd0R\x1d\x06B\x95\xea\\\x1ftP\xff\xff\xff\xff\x94^\'\x01#oYV\xc5\\\xb7@\xff\xff\xff\xff !\x11\x00\x8b\xf3[\xde\xa2\x01\x9dl\xff\xff\xff\xff\xe7z\x92\xc3\x03\xd3\xb5B5 \xaa7\xff\xff\xff\xff\xff\xc3Q:\xa6\xb3\xa3\x1e\x90 \xa0\\\xff\xff\xff\xff\xff\x14<\x03Vr\xe8Z.Ql\xf5\xff\xff\xff\xffEx\xf7\x0b_\xa1\x7f\xfcG\xa4\x18\xcd\xff\xff\xff\xff1|~i\x00\xb3. ,1Q\x0e\xff\xff\xff\xff\x87a\x933Y\xd7\xe1B#\xa7a\xee\xff\xff\xff\xff\r\tJ\x18\xd0\xdb\x0b\xbe\x00\x91\x95\x9e\xff\xff\xff\xffHfW\xcd\x8f\xa9z6|\xc5\x171\xff\xff\xff\xff\xf5tP7\x93\x02Q|x\x17\xb1\xcb\xff\xff\xff\xffVb\x11\xa0*\xd9;\x0b\xf8\x1c\xd3c\xff\xff\xff\xff\x84\x82\xcer\x15\x99`5LmA\xd5\xff\xff\xff\xfft\xce\x8e\xcbw\xee\xf3\xc0w\xca\xb3\xfd\xff\xff\xff\xff\xb0\xaab\x92\xd4\x02\x84H3\x94\xa9~\xff\xff\xff\xff\xfe7\x18\xcaW=\x94\xbc|\x0f{\x84\xff\xff\xff\xff\xe8\xdf\xde?\x8b\xb7\x9dH3\xc1\xf2\xaa\xff\xff\xff\xff\xbe\x00\xba\xd7\xba6!\x95g\xb01\xf9\xff\xff\xff\xff\x93\xe3\x90YH9g\xf7\x97nhv\xff\xff\xff\xff\x82\xc7`\xaebn\x9d\x1e}\xba\x1e/\xff\xff\xff\xff\xbd\xe5\xa1\x05\x03\xf26\xa0\xe2\xc1*\x07\xff\xff\xff\xffny\x88\x9f\x19\xd2\xd0\xf7\x1de\xa7\xe0\xff\xff\xff\xff\xc4O&\x14\x8dVH\x90\x8b+\x03\xf9\xff\xff\xff\xff\xf69\xc2\xabo%\xcc/\xc9\xe4dP\xff\xff\xff\xff (\x08G\xebM\x03\x99Y\xb4\xb3\x1f\xff\xff\xff\xffzH\xd2\x19p#\xc5\xa4)\xfd\x05\x9a\xff\xff\xff\xffd\x86\xb2F\x15\x0f\xf4.\xfd\\\xd4#\xff\xff\xff\xff\xaf\xbe\xc6\x9di\xa0\xbc\xd5>cp\xe2\xff\xff\xff\xff&h\x91\xe9\xa0H\xdd\xaer\x87\x18E\xff\xff\xff\xffjg\x08E\x8f\xa4&\xab\xff\x98\x0ei\xff\xff\xff\xff\x01\xfd{"\xed\\\xa3M\x9e\xc3\xf8K\xff\xff\xff\xff\x87Y\x98T\xf0\xa6\xec\x98\xb3\xef\xa7\xaa\xff\xff\xff\xffA\xced\xfal\xd3\xd9\x06\xc6~\xee}\xff\xff\xff\xff:\x7f\xa2\x10\xc7\xadB,}PF%\xff\xff\xff\xff\xb2\xbc\n\x17%\x98\x904\x89\tF\x1f\xff\xff\xff\xff\xdc\xd8\xc6@#M\x87uf\x02\xc6g\xff\xff\xff\xffK\xaf\xb0-=l\x07\xe1Nv\xe4\xf4\xff\xff\xff\xff\xdb\x13\'Ne\xb2UT\x9a#\xb1^\xff\xff\xff\xff\xb2\rne\xd1\x9d\x88\xda\xbb!\xfa@\xff\xff\xff\xffflq\x0f\x01z]uh\'|?\xff\xff\xff\xff\xd5\'\x19\x865\xba\xf2\xe7\x8fR-\xcc\xff\xff\xff\xff\xce\xd6\xfdi\x04\x9b\xa7\tu\x05\xb7\xc8\xff\xff\xff\xff\xc3\xd0)\x11\xdd\xb1\xa5kp\xc9\xd5\xf7\xff\xff\xff\xff\xffU\x9f \xb7\xa1#3rup[\xff\xff\xff\xff\xfc=' # noqa + """Test xml submissions that contain unicode characters.""" + xml = "v\xee\xf3\xc0k\x91\x91\xae\xff\xff\xff\xff\xcf[$b\xd0\xc9'uW\x80RP\xff\xff\xff\xff7\xd0\x03%F\xa7p\xa2\x87\xb6f\xb1\xff\xff\xff\xffg~\xf3O\xf3\x9b\xbc\xf6ej_$\xff\xff\xff\xff\x13\xe8\xa9D\xed\xfb\xe7\xa4d\x96>\xfa\xff\xff\xff\xff\xc7h\"\x86\x14\\.\xdb\x8aoF\xa4\xff\xff\xff\xff\xcez\xff\x01\x0c\x9a\x94\x18\xe1\x03\x8e\xfa\xff\xff\xff\xff39P|\xf9n\x18F\xb1\xcb\xacd\xff\xff\xff\xff\xce>\x97i;1u\xcfI*\xf2\x8e\xff\xff\xff\xffFg\x9d\x0fR:\xcd*\x14\x85\xf0e\xff\xff\xff\xff\xd6\xdc\xda\x8eM\x06\xf1\xfc\xc1\xe8\xd6\xe0\xff\xff\xff\xff\xe7G\xe1\xa1l\x02T\n\xde\x1boJ\xff\xff\xff\xffz \x92\xbc\tR{#\xbb\x9f\xa6s\xff\xff\xff\xff\xa2\x8f(\xb6=\xe11\xfcV\xcf\xef\x0b\xff\xff\xff\xff\xa3\x83\x7ft\xd7\x05+)\xeb9\\*\xff\xff\xff\xff\xfe\x93\xb2\xa2\x06n;\x1b4\xaf\xa6\x93\xff\xff\xff\xff\xe7\xf7\x12Q\x83\xbb\x9a\xc8\xc8q34\xff\xff\xff\xffT2\xa5\x07\x9a\xc9\x89\xf8\x14Y\xab\x19\xff\xff\xff\xff\x16\xd0R\x1d\x06B\x95\xea\\\x1ftP\xff\xff\xff\xff\x94^'\x01#oYV\xc5\\\xb7@\xff\xff\xff\xff !\x11\x00\x8b\xf3[\xde\xa2\x01\x9dl\xff\xff\xff\xff\xe7z\x92\xc3\x03\xd3\xb5B5 \xaa7\xff\xff\xff\xff\xff\xc3Q:\xa6\xb3\xa3\x1e\x90 \xa0\\\xff\xff\xff\xff\xff\x14<\x03Vr\xe8Z.Ql\xf5\xff\xff\xff\xffEx\xf7\x0b_\xa1\x7f\xfcG\xa4\x18\xcd\xff\xff\xff\xff1|~i\x00\xb3. ,1Q\x0e\xff\xff\xff\xff\x87a\x933Y\xd7\xe1B#\xa7a\xee\xff\xff\xff\xff\r\tJ\x18\xd0\xdb\x0b\xbe\x00\x91\x95\x9e\xff\xff\xff\xffHfW\xcd\x8f\xa9z6|\xc5\x171\xff\xff\xff\xff\xf5tP7\x93\x02Q|x\x17\xb1\xcb\xff\xff\xff\xffVb\x11\xa0*\xd9;\x0b\xf8\x1c\xd3c\xff\xff\xff\xff\x84\x82\xcer\x15\x99`5LmA\xd5\xff\xff\xff\xfft\xce\x8e\xcbw\xee\xf3\xc0w\xca\xb3\xfd\xff\xff\xff\xff\xb0\xaab\x92\xd4\x02\x84H3\x94\xa9~\xff\xff\xff\xff\xfe7\x18\xcaW=\x94\xbc|\x0f{\x84\xff\xff\xff\xff\xe8\xdf\xde?\x8b\xb7\x9dH3\xc1\xf2\xaa\xff\xff\xff\xff\xbe\x00\xba\xd7\xba6!\x95g\xb01\xf9\xff\xff\xff\xff\x93\xe3\x90YH9g\xf7\x97nhv\xff\xff\xff\xff\x82\xc7`\xaebn\x9d\x1e}\xba\x1e/\xff\xff\xff\xff\xbd\xe5\xa1\x05\x03\xf26\xa0\xe2\xc1*\x07\xff\xff\xff\xffny\x88\x9f\x19\xd2\xd0\xf7\x1de\xa7\xe0\xff\xff\xff\xff\xc4O&\x14\x8dVH\x90\x8b+\x03\xf9\xff\xff\xff\xff\xf69\xc2\xabo%\xcc/\xc9\xe4dP\xff\xff\xff\xff (\x08G\xebM\x03\x99Y\xb4\xb3\x1f\xff\xff\xff\xffzH\xd2\x19p#\xc5\xa4)\xfd\x05\x9a\xff\xff\xff\xffd\x86\xb2F\x15\x0f\xf4.\xfd\\\xd4#\xff\xff\xff\xff\xaf\xbe\xc6\x9di\xa0\xbc\xd5>cp\xe2\xff\xff\xff\xff&h\x91\xe9\xa0H\xdd\xaer\x87\x18E\xff\xff\xff\xffjg\x08E\x8f\xa4&\xab\xff\x98\x0ei\xff\xff\xff\xff\x01\xfd{\"\xed\\\xa3M\x9e\xc3\xf8K\xff\xff\xff\xff\x87Y\x98T\xf0\xa6\xec\x98\xb3\xef\xa7\xaa\xff\xff\xff\xffA\xced\xfal\xd3\xd9\x06\xc6~\xee}\xff\xff\xff\xff:\x7f\xa2\x10\xc7\xadB,}PF%\xff\xff\xff\xff\xb2\xbc\n\x17%\x98\x904\x89\tF\x1f\xff\xff\xff\xff\xdc\xd8\xc6@#M\x87uf\x02\xc6g\xff\xff\xff\xffK\xaf\xb0-=l\x07\xe1Nv\xe4\xf4\xff\xff\xff\xff\xdb\x13'Ne\xb2UT\x9a#\xb1^\xff\xff\xff\xff\xb2\rne\xd1\x9d\x88\xda\xbb!\xfa@\xff\xff\xff\xffflq\x0f\x01z]uh'|?\xff\xff\xff\xff\xd5'\x19\x865\xba\xf2\xe7\x8fR-\xcc\xff\xff\xff\xff\xce\xd6\xfdi\x04\x9b\xa7\tu\x05\xb7\xc8\xff\xff\xff\xff\xc3\xd0)\x11\xdd\xb1\xa5kp\xc9\xd5\xf7\xff\xff\xff\xff\xffU\x9f \xb7\xa1#3rup[\xff\xff\xff\xff\xfc=" # noqa - request = RequestFactory().post('/') + request = RequestFactory().post("/") request.user = self.user error, instance = safe_create_instance( - self.user.username, TempFileProxy(xml), None, None, request) - text = 'Improperly formatted XML.' + self.user.username, TempFileProxy(xml), None, None, request + ) + text = "Improperly formatted XML." self.assertContains(error, text, status_code=400) diff --git a/onadata/apps/logger/tests/test_transfer_project_command.py b/onadata/apps/logger/tests/test_transfer_project_command.py index f771d58585..06df9a4567 100644 --- a/onadata/apps/logger/tests/test_transfer_project_command.py +++ b/onadata/apps/logger/tests/test_transfer_project_command.py @@ -4,7 +4,7 @@ from django.contrib.auth import get_user_model from django.core.management import call_command -from django.utils.six import StringIO +from six import StringIO from onadata.apps.logger.models import Project, XForm from onadata.apps.main.tests.test_base import TestBase @@ -12,87 +12,94 @@ class TestMoveProjectToAnewOwner(TestBase): # pylint: disable=C0111 def test_successful_project_transfer(self): # pylint: disable=C0103 - """"Test for a successful project transfer.""" + """ "Test for a successful project transfer.""" user_model = get_user_model() user_1_data = { - 'username': 'user1', - 'email': 'user1@test.com', - 'password': 'test_pass' + "username": "user1", + "email": "user1@test.com", + "password": "test_pass", } user_2_data = { - 'username': 'user2', - 'email': 'user2@test.com', - 'password': 'test_pass' + "username": "user2", + "email": "user2@test.com", + "password": "test_pass", } user1 = user_model.objects.create_user(**user_1_data) user2 = user_model.objects.create_user(**user_2_data) Project.objects.create( - name='Test_project_1', organization=user1, created_by=user1) + name="Test_project_1", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_2', organization=user1, created_by=user1) + name="Test_project_2", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_3', organization=user1, created_by=user1) + name="Test_project_3", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_4', organization=user1, created_by=user1) + name="Test_project_4", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_5', organization=user1, created_by=user1) + name="Test_project_5", organization=user1, created_by=user1 + ) mock_stdout = StringIO() sys.stdout = mock_stdout call_command( - 'transferproject', current_owner='user1', new_owner='user2', - all_projects=True, stdout=mock_stdout + "transferproject", + current_owner="user1", + new_owner="user2", + all_projects=True, + stdout=mock_stdout, ) - expected_output = 'Projects transferred successfully' + expected_output = "Projects transferred successfully" self.assertIn(expected_output, mock_stdout.getvalue()) - self.assertEqual( - 0, Project.objects.filter(organization=user1).count() - ) - self.assertEqual( - 5, Project.objects.filter(organization=user2).count() - ) + self.assertEqual(0, Project.objects.filter(organization=user1).count()) + self.assertEqual(5, Project.objects.filter(organization=user2).count()) def test_single_project_transfer(self): - """"Test for a successful project transfer.""" + """ "Test for a successful project transfer.""" user_model = get_user_model() user_1_data = { - 'username': 'user1', - 'email': 'user1@test.com', - 'password': 'test_pass' + "username": "user1", + "email": "user1@test.com", + "password": "test_pass", } user_2_data = { - 'username': 'user2', - 'email': 'user2@test.com', - 'password': 'test_pass' + "username": "user2", + "email": "user2@test.com", + "password": "test_pass", } user1 = user_model.objects.create_user(**user_1_data) user2 = user_model.objects.create_user(**user_2_data) Project.objects.create( - name='Test_project_1', organization=user1, created_by=user1) + name="Test_project_1", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_2', organization=user1, created_by=user1) + name="Test_project_2", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_3', organization=user1, created_by=user1) + name="Test_project_3", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_4', organization=user1, created_by=user1) + name="Test_project_4", organization=user1, created_by=user1 + ) test_project = Project.objects.create( - name='Test_project_5', organization=user1, created_by=user1) + name="Test_project_5", organization=user1, created_by=user1 + ) mock_stdout = StringIO() sys.stdout = mock_stdout self.assertIsNotNone(test_project.id) call_command( - 'transferproject', current_owner='user1', new_owner='user2', - project_id=test_project.id + "transferproject", + current_owner="user1", + new_owner="user2", + project_id=test_project.id, ) - expected_output = 'Projects transferred successfully' + expected_output = "Projects transferred successfully" self.assertIn(expected_output, mock_stdout.getvalue()) - self.assertEqual( - 4, Project.objects.filter(organization=user1).count() - ) - self.assertEqual( - 1, Project.objects.filter(organization=user2).count() - ) + self.assertEqual(4, Project.objects.filter(organization=user1).count()) + self.assertEqual(1, Project.objects.filter(organization=user2).count()) test_project_refetched = Project.objects.get(id=test_project.id) self.assertEqual(user2, test_project_refetched.organization) @@ -100,12 +107,12 @@ def test_xforms_are_transferred_as_well(self): # pylint: disable=C0103 """Test the transfer of ownership of the XForms.""" xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/tutorial.xlsx" + "../fixtures/tutorial/tutorial.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) self._make_submission(xml_submission_file_path) @@ -113,22 +120,22 @@ def test_xforms_are_transferred_as_well(self): # pylint: disable=C0103 user_model = get_user_model() user_data = { - 'username': 'user', - 'email': 'user@test.com', - 'password': 'test_pass' + "username": "user", + "email": "user@test.com", + "password": "test_pass", } new_owner = user_model.objects.create_user(**user_data) mock_stdout = StringIO() sys.stdout = mock_stdout call_command( - 'transferproject', current_owner='bob', new_owner='user', - all_projects=True, stdout=mock_stdout + "transferproject", + current_owner="bob", + new_owner="user", + all_projects=True, + stdout=mock_stdout, ) - self.assertIn( - 'Projects transferred successfully\n', - mock_stdout.getvalue() - ) - bob = user_model.objects.get(username='bob') + self.assertIn("Projects transferred successfully\n", mock_stdout.getvalue()) + bob = user_model.objects.get(username="bob") bobs_forms = XForm.objects.filter(user=bob) new_owner_forms = XForm.objects.filter(user=new_owner) self.assertEqual(0, bobs_forms.count()) @@ -142,14 +149,16 @@ class TestUserValidation(TestBase): it's stdout is interfering with the other functions causing them to fail. stdout.flush() does not help. """ - def test_user_given_does_not_exist(self): # pylint: disable=C0103 + + def test_user_given_does_not_exist(self): # pylint: disable=C0103 """Test that users are validated before initiating project transfer""" mock_stdout = StringIO() sys.stdout = mock_stdout call_command( - 'transferproject', current_owner='user1', new_owner='user2', - all_projects=True + "transferproject", + current_owner="user1", + new_owner="user2", + all_projects=True, ) - expected_output = 'User user1 does not existUser user2 does '\ - 'not exist\n' + expected_output = "User user1 does not existUser user2 does " "not exist\n" self.assertEqual(mock_stdout.getvalue(), expected_output) diff --git a/onadata/apps/logger/tests/test_update_xform_uuid.py b/onadata/apps/logger/tests/test_update_xform_uuid.py index 10ed529eb6..3752708f64 100644 --- a/onadata/apps/logger/tests/test_update_xform_uuid.py +++ b/onadata/apps/logger/tests/test_update_xform_uuid.py @@ -14,7 +14,8 @@ def setUp(self): # self.csv_filepath = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "fixtures", "test_update_xform_uuids.csv" + "fixtures", + "test_update_xform_uuids.csv", ) # get the last defined uuid with open(self.csv_filepath, "r") as f: @@ -33,8 +34,7 @@ def test_fail_update_on_duplicate_uuid(self): self.xform.uuid = self.new_uuid self.xform.save() try: - update_xform_uuid(self.user.username, self.xform.id_string, - self.new_uuid) + update_xform_uuid(self.user.username, self.xform.id_string, self.new_uuid) except DuplicateUUIDError: self.assertTrue(True) else: diff --git a/onadata/apps/logger/tests/test_webforms.py b/onadata/apps/logger/tests/test_webforms.py index 036a5d5fa7..4185abe0b2 100644 --- a/onadata/apps/logger/tests/test_webforms.py +++ b/onadata/apps/logger/tests/test_webforms.py @@ -11,7 +11,7 @@ from onadata.libs.utils.logger_tools import inject_instanceid -@urlmatch(netloc=r'(.*\.)?enketo\.ona\.io$') +@urlmatch(netloc=r"(.*\.)?enketo\.ona\.io$") def enketo_edit_mock(url, request): response = requests.Response() response.status_code = 201 @@ -25,30 +25,36 @@ def setUp(self): self._publish_transportation_form_and_submit_instance() def __load_fixture(self, *path): - with open(os.path.join(os.path.dirname(__file__), *path), 'r') as f: + with open(os.path.join(os.path.dirname(__file__), *path), "r") as f: return f.read() def test_edit_url(self): - instance = Instance.objects.order_by('id').reverse()[0] - edit_url = reverse(edit_data, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - 'data_id': instance.id - }) + instance = Instance.objects.order_by("id").reverse()[0] + edit_url = reverse( + edit_data, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + "data_id": instance.id, + }, + ) with HTTMock(enketo_edit_mock): response = self.client.get(edit_url) self.assertEqual(response.status_code, 302) - self.assertEqual(response['location'], - 'https://hmh2a.enketo.ona.io') + self.assertEqual(response["location"], "https://hmh2a.enketo.ona.io") def test_inject_instanceid(self): """ Test that 1 and only 1 instance id exists or is injected """ instance = Instance.objects.all().reverse()[0] - xml_str = self.__load_fixture("..", "fixtures", "tutorial", - "instances", - "tutorial_2012-06-27_11-27-53.xml") + xml_str = self.__load_fixture( + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53.xml", + ) # test that we dont have an instance id uuid = get_uuid_from_xml(xml_str) self.assertIsNone(uuid) @@ -59,21 +65,19 @@ def test_inject_instanceid(self): def test_dont_inject_instanceid_if_exists(self): xls_file_path = os.path.join( - os.path.dirname(__file__), - '..', - 'fixtures', - 'tutorial', - 'tutorial.xlsx') + os.path.dirname(__file__), "..", "fixtures", "tutorial", "tutorial.xlsx" + ) self._publish_xls_file_and_set_xform(xls_file_path) xml_file_path = os.path.join( os.path.dirname(__file__), - '..', - 'fixtures', - 'tutorial', - 'instances', - 'tutorial_2012-06-27_11-27-53_w_uuid.xml') + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", + ) self._make_submission(xml_file_path) - instance = Instance.objects.order_by('id').reverse()[0] + instance = Instance.objects.order_by("id").reverse()[0] injected_xml_str = inject_instanceid(instance.xml, instance.uuid) # check that the xml is unmodified self.assertEqual(instance.xml, injected_xml_str) diff --git a/onadata/apps/logger/views.py b/onadata/apps/logger/views.py index cef962db4c..9325a29e0d 100644 --- a/onadata/apps/logger/views.py +++ b/onadata/apps/logger/views.py @@ -2,30 +2,32 @@ """ logger views. """ -import json import os import tempfile -from builtins import str as text from datetime import datetime import pytz +import six from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.contrib.sites.models import Site from django.core.files import File from django.core.files.storage import get_storage_class -from django.http import (HttpResponse, HttpResponseBadRequest, - HttpResponseForbidden, HttpResponseRedirect) +from django.http import ( + HttpResponse, + HttpResponseBadRequest, + HttpResponseForbidden, + HttpResponseRedirect, + JsonResponse, +) from django.shortcuts import get_object_or_404, render from django.template import RequestContext, loader from django.urls import reverse -from django.utils import six -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.views.decorators.csrf import csrf_exempt -from django.views.decorators.http import (require_GET, require_http_methods, - require_POST) +from django.views.decorators.http import require_GET, require_http_methods, require_POST from django_digest import HttpDigestAuthenticator from onadata.apps.logger.import_tools import import_instances_from_zip @@ -36,34 +38,45 @@ from onadata.libs.exceptions import EnketoError from onadata.libs.utils.decorators import is_owner from onadata.libs.utils.log import Actions, audit_log -from onadata.libs.utils.cache_tools import (cache, - USER_PROFILE_PREFIX) +from onadata.libs.utils.cache_tools import cache, USER_PROFILE_PREFIX from onadata.libs.utils.logger_tools import ( - BaseOpenRosaResponse, OpenRosaResponse, OpenRosaResponseBadRequest, - PublishXForm, inject_instanceid, publish_form, remove_xform, - response_with_mimetype_and_name, safe_create_instance) + BaseOpenRosaResponse, + OpenRosaResponse, + OpenRosaResponseBadRequest, + PublishXForm, + inject_instanceid, + publish_form, + remove_xform, + response_with_mimetype_and_name, + safe_create_instance, +) from onadata.libs.utils.user_auth import ( - HttpResponseNotAuthorized, add_cors_headers, has_edit_permission, - has_permission, helper_auth_helper) -from onadata.libs.utils.viewer_tools import ( - get_enketo_urls, get_form, get_form_url) + HttpResponseNotAuthorized, + add_cors_headers, + has_edit_permission, + has_permission, + helper_auth_helper, +) +from onadata.libs.utils.viewer_tools import get_enketo_urls, get_form, get_form_url -IO_ERROR_STRINGS = [ - 'request data read error', 'error during read(65536) on wsgi.input' -] +IO_ERROR_STRINGS = ["request data read error", "error during read(65536) on wsgi.input"] + +# pylint: disable=invalid-name +User = get_user_model() def _bad_request(e): - strerror = text(e) + strerror = str(e) return strerror and strerror in IO_ERROR_STRINGS -def _extract_uuid(text): - text = text[text.find("@key="):-1].replace("@key=", "") - if text.startswith("uuid:"): - text = text.replace("uuid:", "") - return text +def _extract_uuid(input_string): + key_index = input_string.find("@key=") + input_string = input_string[key_index:-1].replace("@key=", "") + if input_string.startswith("uuid:"): + input_string = input_string.replace("uuid:", "") + return input_string def _parse_int(num): @@ -71,28 +84,28 @@ def _parse_int(num): return num and int(num) except ValueError: pass + return None def _html_submission_response(request, instance): data = {} - data['username'] = instance.xform.user.username - data['id_string'] = instance.xform.id_string - data['domain'] = Site.objects.get(id=settings.SITE_ID).domain + data["username"] = instance.xform.user.username + data["id_string"] = instance.xform.id_string + data["domain"] = Site.objects.get(id=settings.SITE_ID).domain return render(request, "submission.html", data) def _submission_response(instance): data = {} - data['message'] = _("Successful submission.") - data['formid'] = instance.xform.id_string - data['encrypted'] = instance.xform.encrypted - data['instanceID'] = u'uuid:%s' % instance.uuid - data['submissionDate'] = instance.date_created.isoformat() - data['markedAsCompleteDate'] = instance.date_modified.isoformat() + data["message"] = _("Successful submission.") + data["formid"] = instance.xform.id_string + data["encrypted"] = instance.xform.encrypted + data["instanceID"] = f"uuid:{instance.uuid}" + data["submissionDate"] = instance.date_created.isoformat() + data["markedAsCompleteDate"] = instance.date_modified.isoformat() - return BaseOpenRosaResponse( - loader.get_template('submission.xml').render(data)) + return BaseOpenRosaResponse(loader.get_template("submission.xml").render(data)) @require_POST @@ -112,25 +125,30 @@ def bulksubmission(request, username): temp_postfile = request.FILES.pop("zip_submission_file", []) except IOError: return HttpResponseBadRequest( - _(u"There was a problem receiving your " - u"ODK submission. [Error: IO Error " - u"reading data]")) + _( + "There was a problem receiving your " + "ODK submission. [Error: IO Error " + "reading data]" + ) + ) if len(temp_postfile) != 1: return HttpResponseBadRequest( - _(u"There was a problem receiving your" - u" ODK submission. [Error: multiple " - u"submission files (?)]")) + _( + "There was a problem receiving your" + " ODK submission. [Error: multiple " + "submission files (?)]" + ) + ) postfile = temp_postfile[0] tempdir = tempfile.gettempdir() our_tfpath = os.path.join(tempdir, postfile.name) - with open(our_tfpath, 'wb') as f: + with open(our_tfpath, "wb") as f: f.write(postfile.read()) - with open(our_tfpath, 'rb') as f: - total_count, success_count, errors = import_instances_from_zip( - f, posting_user) + with open(our_tfpath, "rb") as f: + total_count, success_count, errors = import_instances_from_zip(f, posting_user) # chose the try approach as suggested by the link below # http://stackoverflow.com/questions/82831 try: @@ -138,22 +156,31 @@ def bulksubmission(request, username): except IOError: pass json_msg = { - 'message': _(u"Submission complete. Out of %(total)d " - u"survey instances, %(success)d were imported, " - u"(%(rejected)d were rejected as duplicates, " - u"missing forms, etc.)") % { - 'total': total_count, - 'success': success_count, - 'rejected': total_count - success_count - }, - 'errors': u"%d %s" % (len(errors), errors) + "message": _( + "Submission complete. Out of %(total)d " + "survey instances, %(success)d were imported, " + "(%(rejected)d were rejected as duplicates, " + "missing forms, etc.)" + ) + % { + "total": total_count, + "success": success_count, + "rejected": total_count - success_count, + }, + "errors": f"{len(errors)} {errors}", } audit = {"bulk_submission_log": json_msg} - audit_log(Actions.USER_BULK_SUBMISSION, request.user, posting_user, - _("Made bulk submissions."), audit, request) - response = HttpResponse(json.dumps(json_msg)) + audit_log( + Actions.USER_BULK_SUBMISSION, + request.user, + posting_user, + _("Made bulk submissions."), + audit, + request, + ) + response = JsonResponse(json_msg) response.status_code = 200 - response['Location'] = request.build_absolute_uri(request.path) + response["Location"] = request.build_absolute_uri(request.path) return response @@ -164,22 +191,23 @@ def bulksubmission_form(request, username=None): """ username = username if username is None else username if request.user.username == username: - return render(request, 'bulk_submission_form.html') + return render(request, "bulk_submission_form.html") - return HttpResponseRedirect('/%s' % request.user.username) + return HttpResponseRedirect(f"/{request.user.username}") +# pylint: disable=invalid-name @require_GET -def formList(request, username): # pylint: disable=C0103 +def formList(request, username): # noqa N802 """ formList view, /formList OpenRosa Form Discovery API 1.0. """ formlist_user = get_object_or_404(User, username__iexact=username) - profile = cache.get( - f'{USER_PROFILE_PREFIX}{formlist_user.username}') + profile = cache.get(f"{USER_PROFILE_PREFIX}{formlist_user.username}") if not profile: profile, __ = UserProfile.objects.get_or_create( - user__username=formlist_user.username) + user__username=formlist_user.username + ) if profile.require_auth: authenticator = HttpDigestAuthenticator() @@ -195,54 +223,56 @@ def formList(request, username): # pylint: disable=C0103 # for users who are non-owner if request.user.username == profile.user.username: xforms = XForm.objects.filter( - downloadable=True, - deleted_at__isnull=True, - user__username__iexact=username) + downloadable=True, deleted_at__isnull=True, user__username__iexact=username + ) else: xforms = XForm.objects.filter( downloadable=True, deleted_at__isnull=True, user__username__iexact=username, - require_auth=False) + require_auth=False, + ) audit = {} - audit_log(Actions.USER_FORMLIST_REQUESTED, request.user, formlist_user, - _("Requested forms list."), audit, request) + audit_log( + Actions.USER_FORMLIST_REQUESTED, + request.user, + formlist_user, + _("Requested forms list."), + audit, + request, + ) data = { - 'host': request.build_absolute_uri().replace(request.get_full_path(), - ''), - 'xforms': xforms + "host": request.build_absolute_uri().replace(request.get_full_path(), ""), + "xforms": xforms, } response = render( - request, - "xformsList.xml", - data, - content_type="text/xml; charset=utf-8") - response['X-OpenRosa-Version'] = '1.0' - response['Date'] = datetime.now(pytz.timezone(settings.TIME_ZONE))\ - .strftime('%a, %d %b %Y %H:%M:%S %Z') + request, "xformsList.xml", data, content_type="text/xml; charset=utf-8" + ) + response["X-OpenRosa-Version"] = "1.0" + response["Date"] = datetime.now(pytz.timezone(settings.TIME_ZONE)).strftime( + "%a, %d %b %Y %H:%M:%S %Z" + ) return response +# pylint: disable=invalid-name @require_GET -def xformsManifest(request, username, id_string): # pylint: disable=C0103 +def xformsManifest(request, username, id_string): # noqa N802 """ XFormManifest view, part of OpenRosa Form Discovery API 1.0. """ - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } + xform_kwargs = {"id_string__iexact": id_string, "user__username__iexact": username} xform = get_form(xform_kwargs) formlist_user = xform.user - profile = cache.get( - f'{USER_PROFILE_PREFIX}{formlist_user.username}') + profile = cache.get(f"{USER_PROFILE_PREFIX}{formlist_user.username}") if not profile: profile, __ = UserProfile.objects.get_or_create( - user__username=formlist_user.username) + user__username=formlist_user.username + ) if profile.require_auth: authenticator = HttpDigestAuthenticator() @@ -251,23 +281,26 @@ def xformsManifest(request, username, id_string): # pylint: disable=C0103 response = render( request, - "xformsManifest.xml", { - 'host': - request.build_absolute_uri().replace(request.get_full_path(), ''), - 'media_files': - MetaData.media_upload(xform, download=True) + "xformsManifest.xml", + { + "host": request.build_absolute_uri().replace(request.get_full_path(), ""), + "media_files": MetaData.media_upload(xform, download=True), }, - content_type="text/xml; charset=utf-8") - response['X-OpenRosa-Version'] = '1.0' - response['Date'] = datetime.now(pytz.timezone(settings.TIME_ZONE))\ - .strftime('%a, %d %b %Y %H:%M:%S %Z') + content_type="text/xml; charset=utf-8", + ) + response["X-OpenRosa-Version"] = "1.0" + response["Date"] = datetime.now(pytz.timezone(settings.TIME_ZONE)).strftime( + "%a, %d %b %Y %H:%M:%S %Z" + ) return response +# pylint: disable=too-many-return-statements +# pylint: disable=too-many-branches @require_http_methods(["HEAD", "POST"]) @csrf_exempt -def submission(request, username=None): # pylint: disable=R0911,R0912 +def submission(request, username=None): # noqa C901 """ Submission view, /submission of the OpenRosa Form Submission API 1.0. """ @@ -280,14 +313,16 @@ def submission(request, username=None): # pylint: disable=R0911,R0912 if not authenticator.authenticate(request): return authenticator.build_challenge_response() - if request.method == 'HEAD': + if request.method == "HEAD": response = OpenRosaResponse(status=204) if username: - response['Location'] = request.build_absolute_uri().replace( - request.get_full_path(), '/%s/submission' % username) + response["Location"] = request.build_absolute_uri().replace( + request.get_full_path(), f"/{username}/submission" + ) else: - response['Location'] = request.build_absolute_uri().replace( - request.get_full_path(), '/submission') + response["Location"] = request.build_absolute_uri().replace( + request.get_full_path(), "/submission" + ) return response xml_file_list = [] @@ -299,27 +334,33 @@ def submission(request, username=None): # pylint: disable=R0911,R0912 xml_file_list = request.FILES.pop("xml_submission_file", []) if len(xml_file_list) != 1: return OpenRosaResponseBadRequest( - _(u"There should be a single XML submission file.")) + _("There should be a single XML submission file.") + ) # save this XML file and media files as attachments media_files = request.FILES.values() # get uuid from post request - uuid = request.POST.get('uuid') + uuid = request.POST.get("uuid") - error, instance = safe_create_instance(username, xml_file_list[0], - media_files, uuid, request) + error, instance = safe_create_instance( + username, xml_file_list[0], media_files, uuid, request + ) if error: return error - elif instance is None: - return OpenRosaResponseBadRequest( - _(u"Unable to create submission.")) + if instance is None: + return OpenRosaResponseBadRequest(_("Unable to create submission.")) audit = {"xform": instance.xform.id_string} - audit_log(Actions.SUBMISSION_CREATED, request.user, - instance.xform.user, - _("Created submission on form %(id_string)s.") % - {"id_string": instance.xform.id_string}, audit, request) + audit_log( + Actions.SUBMISSION_CREATED, + request.user, + instance.xform.user, + _("Created submission on form %(id_string)s.") + % {"id_string": instance.xform.id_string}, + audit, + request, + ) # response as html if posting with a UUID if not username and uuid: @@ -331,19 +372,19 @@ def submission(request, username=None): # pylint: disable=R0911,R0912 # 1) the status code needs to be 201 (created) # 2) The location header needs to be set to the host it posted to response.status_code = 201 - response['Location'] = request.build_absolute_uri(request.path) + response["Location"] = request.build_absolute_uri(request.path) return response except IOError as e: if _bad_request(e): - return OpenRosaResponseBadRequest( - _(u"File transfer interruption.")) - else: - raise + return OpenRosaResponseBadRequest(_("File transfer interruption.")) + raise finally: if xml_file_list: - [_file.close() for _file in xml_file_list] # pylint: disable=W0106 + for _file in xml_file_list: + _file.close() if media_files: - [_file.close() for _file in media_files] # pylint: disable=W0106 + for _file in media_files: + _file.close() def download_xform(request, username, id_string): @@ -351,7 +392,7 @@ def download_xform(request, username, id_string): Download XForm XML view. """ user = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': user, 'id_string__iexact': id_string}) + xform = get_form({"user": user, "id_string__iexact": id_string}) profile, __ = UserProfile.objects.get_or_create(user=user) if profile.require_auth: @@ -359,11 +400,15 @@ def download_xform(request, username, id_string): if not authenticator.authenticate(request): return authenticator.build_challenge_response() audit = {"xform": xform.id_string} - audit_log(Actions.FORM_XML_DOWNLOADED, request.user, xform.user, - _("Downloaded XML for form '%(id_string)s'.") % - {"id_string": xform.id_string}, audit, request) - response = response_with_mimetype_and_name( - 'xml', id_string, show_date=False) + audit_log( + Actions.FORM_XML_DOWNLOADED, + request.user, + xform.user, + _("Downloaded XML for form '%(id_string)s'.") % {"id_string": xform.id_string}, + audit, + request, + ) + response = response_with_mimetype_and_name("xml", id_string, show_date=False) response.content = xform.xml return response @@ -372,46 +417,53 @@ def download_xlsform(request, username, id_string): """ Download XLSForm view. """ - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) owner = User.objects.get(username__iexact=username) helper_auth_helper(request) if not has_permission(xform, owner, request, xform.shared): - return HttpResponseForbidden('Not shared.') + return HttpResponseForbidden("Not shared.") file_path = xform.xls.name default_storage = get_storage_class()() - if file_path != '' and default_storage.exists(file_path): + if file_path != "" and default_storage.exists(file_path): audit = {"xform": xform.id_string} - audit_log(Actions.FORM_XLS_DOWNLOADED, request.user, xform.user, - _("Downloaded XLS file for form '%(id_string)s'.") % - {"id_string": xform.id_string}, audit, request) + audit_log( + Actions.FORM_XLS_DOWNLOADED, + request.user, + xform.user, + _("Downloaded XLS file for form '%(id_string)s'.") + % {"id_string": xform.id_string}, + audit, + request, + ) split_path = file_path.split(os.extsep) - extension = 'xls' + extension = "xls" if len(split_path) > 1: extension = split_path[len(split_path) - 1] response = response_with_mimetype_and_name( - 'vnd.ms-excel', + "vnd.ms-excel", id_string, show_date=False, extension=extension, - file_path=file_path) + file_path=file_path, + ) return response - else: - messages.add_message(request, messages.WARNING, - _(u'No XLS file for your form ' - u'%(id)s') % {'id': id_string}) + messages.add_message( + request, + messages.WARNING, + _("No XLS file for your form " "%(id)s") % {"id": id_string}, + ) - return HttpResponseRedirect("/%s" % username) + return HttpResponseRedirect(f"/{username}") def download_jsonform(request, username, id_string): @@ -419,10 +471,9 @@ def download_jsonform(request, username, id_string): XForm JSON view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) if request.method == "OPTIONS": response = HttpResponse() @@ -430,14 +481,13 @@ def download_jsonform(request, username, id_string): return response helper_auth_helper(request) if not has_permission(xform, owner, request, xform.shared): - response = HttpResponseForbidden(_(u'Not shared.')) + response = HttpResponseForbidden(_("Not shared.")) add_cors_headers(response) return response - response = response_with_mimetype_and_name( - 'json', id_string, show_date=False) - if 'callback' in request.GET and request.GET.get('callback') != '': - callback = request.GET.get('callback') - response.content = "%s(%s)" % (callback, xform.json) + response = response_with_mimetype_and_name("json", id_string, show_date=False) + if "callback" in request.GET and request.GET.get("callback") != "": + callback = request.GET.get("callback") + response.content = f"{callback}({xform.json})" else: add_cors_headers(response) response.content = xform.json @@ -450,20 +500,26 @@ def delete_xform(request, username, id_string): """ Delete XForm view. """ - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) # delete xform and submissions remove_xform(xform) audit = {} - audit_log(Actions.FORM_DELETED, request.user, xform.user, - _("Deleted form '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) - return HttpResponseRedirect('/') + audit_log( + Actions.FORM_DELETED, + request.user, + xform.user, + _("Deleted form '%(id_string)s'.") + % { + "id_string": xform.id_string, + }, + audit, + request, + ) + return HttpResponseRedirect("/") @is_owner @@ -471,22 +527,27 @@ def toggle_downloadable(request, username, id_string): """ Toggle XForm view, changes downloadable status of a form. """ - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) xform.downloadable = not xform.downloadable xform.save() audit = {} - audit_log(Actions.FORM_UPDATED, request.user, xform.user, - _("Made form '%(id_string)s' %(downloadable)s.") % { - 'id_string': - xform.id_string, - 'downloadable': - _("downloadable") - if xform.downloadable else _("un-downloadable") - }, audit, request) - return HttpResponseRedirect("/%s" % username) + audit_log( + Actions.FORM_UPDATED, + request.user, + xform.user, + _("Made form '%(id_string)s' %(downloadable)s.") + % { + "id_string": xform.id_string, + "downloadable": _("downloadable") + if xform.downloadable + else _("un-downloadable"), + }, + audit, + request, + ) + return HttpResponseRedirect(f"/{username}") def enter_data(request, username, id_string): @@ -494,48 +555,47 @@ def enter_data(request, username, id_string): Redirects to Enketo webform view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) if not has_edit_permission(xform, owner, request, xform.shared): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) form_url = get_form_url(request, username, settings.ENKETO_PROTOCOL) try: enketo_urls = get_enketo_urls(form_url, xform.id_string) - url = enketo_urls.get('url') + url = enketo_urls.get("url") if not url: return HttpResponseRedirect( reverse( - 'form-show', - kwargs={'username': username, - 'id_string': id_string})) + "form-show", kwargs={"username": username, "id_string": id_string} + ) + ) return HttpResponseRedirect(url) except EnketoError as e: data = {} owner = User.objects.get(username__iexact=username) - data['profile'], __ = UserProfile.objects.get_or_create(user=owner) - data['xform'] = xform - data['content_user'] = owner - data['form_view'] = True - data['message'] = { - 'type': 'alert-error', - 'text': u"Enketo error, reason: %s" % e + data["profile"], __ = UserProfile.objects.get_or_create(user=owner) + data["xform"] = xform + data["content_user"] = owner + data["form_view"] = True + data["message"] = { + "type": "alert-error", + "text": f"Enketo error, reason: {e}", } messages.add_message( request, messages.WARNING, - _("Enketo error: enketo replied %s") % e, - fail_silently=True) + _(f"Enketo error: enketo replied {e}"), + fail_silently=True, + ) return render(request, "profile.html", data) return HttpResponseRedirect( - reverse( - 'form-show', kwargs={'username': username, - 'id_string': id_string})) + reverse("form-show", kwargs={"username": username, "id_string": id_string}) + ) def edit_data(request, username, id_string, data_id): @@ -544,30 +604,27 @@ def edit_data(request, username, id_string, data_id): """ context = RequestContext(request) owner = User.objects.get(username__iexact=username) - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } + xform_kwargs = {"id_string__iexact": id_string, "user__username__iexact": username} xform = get_form(xform_kwargs) instance = get_object_or_404(Instance, pk=data_id, xform=xform) if not has_edit_permission(xform, owner, request, xform.shared): - return HttpResponseForbidden(_(u'Not shared.')) - if not hasattr(settings, 'ENKETO_URL'): + return HttpResponseForbidden(_("Not shared.")) + if not hasattr(settings, "ENKETO_URL"): return HttpResponseRedirect( - reverse( - 'form-show', - kwargs={'username': username, - 'id_string': id_string})) + reverse("form-show", kwargs={"username": username, "id_string": id_string}) + ) - url = '%sdata/edit_url' % settings.ENKETO_URL + url = f"{settings.ENKETO_URL}data/edit_url" # see commit 220f2dad0e for tmp file creation injected_xml = inject_instanceid(instance.xml, instance.uuid) return_url = request.build_absolute_uri( reverse( - 'submission-instance', - kwargs={'username': username, - 'id_string': id_string}) + "#/" + text(instance.id)) + "submission-instance", kwargs={"username": username, "id_string": id_string} + ) + + "#/" + + str(instance.id) + ) form_url = get_form_url(request, username, settings.ENKETO_PROTOCOL) try: @@ -576,26 +633,27 @@ def edit_data(request, username, id_string, data_id): xform.id_string, instance_xml=injected_xml, instance_id=instance.uuid, - return_url=return_url) + return_url=return_url, + ) except EnketoError as e: context.message = { - 'type': 'alert-error', - 'text': u"Enketo error, reason: %s" % e + "type": "alert-error", + "text": f"Enketo error, reason: {e}", } messages.add_message( request, messages.WARNING, - _("Enketo error: enketo replied %s") % e, - fail_silently=True) + _(f"Enketo error: enketo replied {e}"), + fail_silently=True, + ) else: if url: - url = url['edit_url'] + url = url["edit_url"] context.enketo = url return HttpResponseRedirect(url) return HttpResponseRedirect( - reverse( - 'form-show', kwargs={'username': username, - 'id_string': id_string})) + reverse("form-show", kwargs={"username": username, "id_string": id_string}) + ) def view_submission_list(request, username): @@ -609,18 +667,15 @@ def view_submission_list(request, username): authenticator = HttpDigestAuthenticator() if not authenticator.authenticate(request): return authenticator.build_challenge_response() - id_string = request.GET.get('formId', None) - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } + id_string = request.GET.get("formId", None) + xform_kwargs = {"id_string__iexact": id_string, "user__username__iexact": username} xform = get_form(xform_kwargs) if not has_permission(xform, form_user, request, xform.shared_data): - return HttpResponseForbidden('Not shared.') - num_entries = request.GET.get('numEntries', None) - cursor = request.GET.get('cursor', None) - instances = xform.instances.filter(deleted_at=None).order_by('pk') + return HttpResponseForbidden("Not shared.") + num_entries = request.GET.get("numEntries", None) + cursor = request.GET.get("cursor", None) + instances = xform.instances.filter(deleted_at=None).order_by("pk") cursor = _parse_int(cursor) if cursor: @@ -630,7 +685,7 @@ def view_submission_list(request, username): if num_entries: instances = instances[:num_entries] - data = {'instances': instances} + data = {"instances": instances} resumption_cursor = 0 if instances.count(): @@ -639,13 +694,11 @@ def view_submission_list(request, username): elif instances.count() == 0 and cursor: resumption_cursor = cursor - data['resumptionCursor'] = resumption_cursor + data["resumptionCursor"] = resumption_cursor return render( - request, - 'submissionList.xml', - data, - content_type="text/xml; charset=utf-8") + request, "submissionList.xml", data, content_type="text/xml; charset=utf-8" + ) def view_download_submission(request, username): @@ -660,12 +713,12 @@ def view_download_submission(request, username): if not authenticator.authenticate(request): return authenticator.build_challenge_response() data = {} - form_id = request.GET.get('formId', None) + form_id = request.GET.get("formId", None) if not isinstance(form_id, six.string_types): return HttpResponseBadRequest() - - id_string = form_id[0:form_id.find('[')] - form_id_parts = form_id.split('/') + last_index = form_id.find("[") + id_string = form_id[0:last_index] + form_id_parts = form_id.split("/") if form_id_parts.__len__() < 2: return HttpResponseBadRequest() @@ -675,25 +728,23 @@ def view_download_submission(request, username): xform__id_string__iexact=id_string, uuid=uuid, xform__user__username=username, - deleted_at__isnull=True) + deleted_at__isnull=True, + ) xform = instance.xform if not has_permission(xform, form_user, request, xform.shared_data): - return HttpResponseForbidden('Not shared.') + return HttpResponseForbidden("Not shared.") submission_xml_root_node = instance.get_root_node() - submission_xml_root_node.setAttribute('instanceID', - u'uuid:%s' % instance.uuid) - submission_xml_root_node.setAttribute('submissionDate', - instance.date_created.isoformat()) - data['submission_data'] = submission_xml_root_node.toxml() - data['media_files'] = Attachment.objects.filter(instance=instance) - data['host'] = request.build_absolute_uri().replace( - request.get_full_path(), '') + submission_xml_root_node.setAttribute("instanceID", f"uuid:{instance.uuid}") + submission_xml_root_node.setAttribute( + "submissionDate", instance.date_created.isoformat() + ) + data["submission_data"] = submission_xml_root_node.toxml() + data["media_files"] = Attachment.objects.filter(instance=instance) + data["host"] = request.build_absolute_uri().replace(request.get_full_path(), "") return render( - request, - 'downloadSubmission.xml', - data, - content_type="text/xml; charset=utf-8") + request, "downloadSubmission.xml", data, content_type="text/xml; charset=utf-8" + ) @require_http_methods(["HEAD", "POST"]) @@ -711,25 +762,26 @@ def form_upload(request, username): return authenticator.build_challenge_response() if form_user != request.user: return HttpResponseForbidden( - _(u"Not allowed to upload form[s] to %(user)s account." % - {'user': form_user})) - if request.method == 'HEAD': + _(f"Not allowed to upload form[s] to {form_user} account.") + ) + if request.method == "HEAD": response = OpenRosaResponse(status=204) - response['Location'] = request.build_absolute_uri().replace( - request.get_full_path(), '/%s/formUpload' % form_user.username) + response["Location"] = request.build_absolute_uri().replace( + request.get_full_path(), f"/{form_user.username}/formUpload" + ) return response - xform_def = request.FILES.get('form_def_file', None) - content = u"" + xform_def = request.FILES.get("form_def_file", None) + content = "" if isinstance(xform_def, File): do_form_upload = PublishXForm(xform_def, form_user) xform = publish_form(do_form_upload.publish_xform) status = 201 if isinstance(xform, XForm): - content = _(u"%s successfully published." % xform.id_string) + content = _(f"{xform.id_string} successfully published.") else: - content = xform['text'] + content = xform["text"] if isinstance(content, Exception): - content = content + content = str(content) status = 500 else: status = 400 diff --git a/onadata/apps/logger/xform_fs.py b/onadata/apps/logger/xform_fs.py index e2173db272..0bc337afe4 100644 --- a/onadata/apps/logger/xform_fs.py +++ b/onadata/apps/logger/xform_fs.py @@ -1,19 +1,30 @@ +# -*- coding: utf-8 -*- +""" +ODK Collect/Briefcase XForm instances folder traversal. +""" import os import glob import re -class XFormInstanceFS(object): +# pylint: disable=too-many-instance-attributes +class XFormInstanceFS: + """A class to traverse an ODK Collect/Briefcase XForm instances folder.""" + def __init__(self, filepath): self.path = filepath self.directory, self.filename = os.path.split(self.path) self.xform_id = re.sub(".xml", "", self.filename) + self._photos = [] + self._osm = [] + self._metadata_directory = "" + self._xml = "" @property def photos(self): - if not hasattr(self, '_photos'): + """Returns all .jpg file paths.""" + if not getattr(self, "_photos"): available_photos = glob.glob(os.path.join(self.directory, "*.jpg")) - self._photos = [] for photo_path in available_photos: _pdir, photo = os.path.split(photo_path) if self.xml.find(photo) > 0: @@ -22,9 +33,9 @@ def photos(self): @property def osm(self): - if not hasattr(self, '_osm'): + """Returns all .osm file paths.""" + if not getattr(self, "_osm"): available_osm = glob.glob(os.path.join(self.directory, "*.osm")) - self._osm = [] for osm_path in available_osm: _pdir, osm = os.path.split(osm_path) if self.xml.find(osm) > 0: @@ -33,34 +44,32 @@ def osm(self): @property def metadata_directory(self): - if not hasattr(self, '_metadata_directory'): - instances_dir = os.path.join( - self.directory, "..", "..", "instances") - metadata_directory = os.path.join( - self.directory, "..", "..", "metadata") - if os.path.exists(instances_dir) and os.path.exists( - metadata_directory): + """Returns the metadata directory.""" + if not getattr(self, "_metadata_directory"): + instances_dir = os.path.join(self.directory, "..", "..", "instances") + metadata_directory = os.path.join(self.directory, "..", "..", "metadata") + if os.path.exists(instances_dir) and os.path.exists(metadata_directory): self._metadata_directory = os.path.abspath(metadata_directory) - else: - self._metadata_directory = False return self._metadata_directory @property def xml(self): - if not hasattr(self, '_xml'): - with open(self.path, 'r') as f: + """Returns the submission XML""" + if not getattr(self, "_xml"): + with open(self.path, "r", encoding="utf-8") as f: self._xml = f.read() return self._xml @classmethod def is_valid_instance(cls, filepath): + """Returns True if the XML at ``filepath`` is a valid XML file.""" if not filepath.endswith(".xml"): return False - with open(filepath, 'r') as ff: - fxml = ff.read() - if not fxml.strip().startswith('" % self.xform_id + return f"" diff --git a/onadata/apps/logger/xform_instance_parser.py b/onadata/apps/logger/xform_instance_parser.py index 652ad42436..bf2e160049 100644 --- a/onadata/apps/logger/xform_instance_parser.py +++ b/onadata/apps/logger/xform_instance_parser.py @@ -1,12 +1,15 @@ +# -*- coding: utf-8 -*- +""" +XForm submission XML parser utility functions. +""" import logging import re -import dateutil.parser -from builtins import str as text -from future.utils import python_2_unicode_compatible from xml.dom import minidom, Node -from django.utils.encoding import smart_text, smart_str -from django.utils.translation import ugettext as _ +import dateutil.parser + +from django.utils.encoding import smart_str +from django.utils.translation import gettext as _ from onadata.libs.utils.common_tags import XFORM_ID_STRING, VERSION @@ -15,28 +18,24 @@ class XLSFormError(Exception): pass -@python_2_unicode_compatible class DuplicateInstance(Exception): def __str__(self): - return _(u'Duplicate Instance') + return _("Duplicate Instance") -@python_2_unicode_compatible class InstanceInvalidUserError(Exception): def __str__(self): - return _(u'Could not determine the user.') + return _("Could not determine the user.") -@python_2_unicode_compatible class InstanceParseError(Exception): def __str__(self): - return _(u'The instance could not be parsed.') + return _("The instance could not be parsed.") -@python_2_unicode_compatible class InstanceEmptyError(InstanceParseError): def __str__(self): - return _(u'Empty instance') + return _("Empty instance") class InstanceFormatError(Exception): @@ -60,6 +59,9 @@ class NonUniqueFormIdError(Exception): def get_meta_from_xml(xml_str, meta_name): + """ + Return the meta section of an XForm submission XML. + """ xml = clean_and_parse_xml(xml_str) children = xml.childNodes # children ideally contains a single element @@ -67,52 +69,71 @@ def get_meta_from_xml(xml_str, meta_name): if children.length == 0: raise ValueError(_("XML string must have a survey element.")) survey_node = children[0] - meta_tags = [n for n in survey_node.childNodes if - n.nodeType == Node.ELEMENT_NODE and - (n.tagName.lower() == "meta" or - n.tagName.lower() == "orx:meta")] + meta_tags = [ + n + for n in survey_node.childNodes + if n.nodeType == Node.ELEMENT_NODE + and (n.tagName.lower() == "meta" or n.tagName.lower() == "orx:meta") + ] if len(meta_tags) == 0: return None # get the requested tag meta_tag = meta_tags[0] - uuid_tags = [n for n in meta_tag.childNodes if - n.nodeType == Node.ELEMENT_NODE and - (n.tagName.lower() == meta_name.lower() or - n.tagName.lower() == u'orx:%s' % meta_name.lower())] + uuid_tags = [ + n + for n in meta_tag.childNodes + if n.nodeType == Node.ELEMENT_NODE + and ( + n.tagName.lower() == meta_name.lower() + or n.tagName.lower() == f"orx:{meta_name.lower()}" + ) + ] if len(uuid_tags) == 0: return None uuid_tag = uuid_tags[0] - return uuid_tag.firstChild.nodeValue.strip() if uuid_tag.firstChild\ - else None + + return uuid_tag.firstChild.nodeValue.strip() if uuid_tag.firstChild else None def get_uuid_from_xml(xml): + """ + Returns the uuid of an XForm submisison XML + """ + def _uuid_only(uuid, regex): matches = regex.match(uuid) if matches and len(matches.groups()) > 0: return matches.groups()[0] return None + uuid = get_meta_from_xml(xml, "instanceID") regex = re.compile(r"uuid:(.*)") if uuid: return _uuid_only(uuid, regex) + # check in survey_node attributes xml = clean_and_parse_xml(xml) children = xml.childNodes + # children ideally contains a single element # that is the parent of all survey elements if children.length == 0: raise ValueError(_("XML string must have a survey element.")) + survey_node = children[0] - uuid = survey_node.getAttribute('instanceID') - if uuid != '': + uuid = survey_node.getAttribute("instanceID") + if uuid != "": return _uuid_only(uuid, regex) + return None def get_submission_date_from_xml(xml): + """ + Returns submissionDate from an XML submission. + """ # check in survey_node attributes xml = clean_and_parse_xml(xml) children = xml.childNodes @@ -121,13 +142,16 @@ def get_submission_date_from_xml(xml): if children.length == 0: raise ValueError(_("XML string must have a survey element.")) survey_node = children[0] - submissionDate = survey_node.getAttribute('submissionDate') - if submissionDate != '': - return dateutil.parser.parse(submissionDate) + submission_date = survey_node.getAttribute("submissionDate") + if submission_date != "": + return dateutil.parser.parse(submission_date) return None def get_deprecated_uuid_from_xml(xml): + """ + Returns the deprecatedID from submission XML + """ uuid = get_meta_from_xml(xml, "deprecatedID") regex = re.compile(r"uuid:(.*)") if uuid: @@ -138,77 +162,82 @@ def get_deprecated_uuid_from_xml(xml): def clean_and_parse_xml(xml_string): + """ + Removes spaces between XML tags in ``xml_string`` + + Returns an XML object via minidom.parseString(xml_string) + """ clean_xml_str = xml_string.strip() - clean_xml_str = re.sub(r">\s+<", u"><", smart_text(clean_xml_str)) + clean_xml_str = re.sub(r">\s+<", "><", smart_str(clean_xml_str)) xml_obj = minidom.parseString(smart_str(clean_xml_str)) return xml_obj -def _xml_node_to_dict(node, repeats=[], encrypted=False): +# pylint: disable=too-many-branches +def _xml_node_to_dict(node, repeats=None, encrypted=False): # noqa C901 + repeats = [] if repeats is None else repeats if len(node.childNodes) == 0: # there's no data for this leaf node return None - elif len(node.childNodes) == 1 and \ - node.childNodes[0].nodeType == node.TEXT_NODE: + if len(node.childNodes) == 1 and node.childNodes[0].nodeType == node.TEXT_NODE: # there is data for this leaf node return {node.nodeName: node.childNodes[0].nodeValue} - else: - # this is an internal node - value = {} - - for child in node.childNodes: - # handle CDATA text section - if child.nodeType == child.CDATA_SECTION_NODE: - return {child.parentNode.nodeName: child.nodeValue} - - d = _xml_node_to_dict(child, repeats) - - if d is None: - continue - - child_name = child.nodeName - child_xpath = xpath_from_xml_node(child) - if list(d) != [child_name]: - raise AssertionError() - node_type = dict - # check if name is in list of repeats and make it a list if so - # All the photo attachments in an encrypted form use name media - if child_xpath in repeats or (encrypted and child_name == 'media'): - node_type = list - - if node_type == dict: - if child_name not in value: - value[child_name] = d[child_name] - else: - # node is repeated, aggregate node values - node_value = value[child_name] - # 1. check if the node values is a list - if not isinstance(node_value, list): - # if not a list create - value[child_name] = [node_value] - # 2. parse the node - d = _xml_node_to_dict(child, repeats) - # 3. aggregate - value[child_name].append(d[child_name]) + # this is an internal node + value = {} + + for child in node.childNodes: + # handle CDATA str section + if child.nodeType == child.CDATA_SECTION_NODE: + return {child.parentNode.nodeName: child.nodeValue} + + child_dict = _xml_node_to_dict(child, repeats) + + if child_dict is None: + continue + + child_name = child.nodeName + child_xpath = xpath_from_xml_node(child) + if list(child_dict) != [child_name]: + raise AssertionError() + node_type = dict + # check if name is in list of repeats and make it a list if so + # All the photo attachments in an encrypted form use name media + if child_xpath in repeats or (encrypted and child_name == "media"): + node_type = list + + if node_type == dict: + if child_name not in value: + value[child_name] = child_dict[child_name] else: - if child_name not in value: - value[child_name] = [d[child_name]] - else: - value[child_name].append(d[child_name]) - if value == {}: - return None + # node is repeated, aggregate node values + node_value = value[child_name] + # 1. check if the node values is a list + if not isinstance(node_value, list): + # if not a list create + value[child_name] = [node_value] + # 2. parse the node + child_dict = _xml_node_to_dict(child, repeats) + # 3. aggregate + value[child_name].append(child_dict[child_name]) else: - return {node.nodeName: value} + if child_name not in value: + value[child_name] = [child_dict[child_name]] + else: + value[child_name].append(child_dict[child_name]) + if not value: + return None + + return {node.nodeName: value} -def _flatten_dict(d, prefix): +def _flatten_dict(data_dict, prefix): """ Return a list of XPath, value pairs. - :param d: A dictionary + :param data_dict: A python dictionary object :param prefix: A list of prefixes """ - for key, value in d.items(): + for key, value in data_dict.items(): new_prefix = prefix + [key] if isinstance(value, dict): @@ -225,7 +254,7 @@ def _flatten_dict(d, prefix): # hack: removing [1] index to be consistent across # surveys that have a single repitition of the # loop versus mutliple. - item_prefix[-1] += u"[%s]" % text(i + 1) + item_prefix[-1] += f"[{str(i + 1)}]" if isinstance(item, dict): for pair in _flatten_dict(item, item_prefix): @@ -236,14 +265,15 @@ def _flatten_dict(d, prefix): yield (new_prefix, value) -def _flatten_dict_nest_repeats(d, prefix): +def _flatten_dict_nest_repeats(data_dict, prefix): """ Return a list of XPath, value pairs. - :param d: A dictionary + :param data_dict: A python dictionary object + :param prefix: A list of prefixes :param prefix: A list of prefixes """ - for key, value in d.items(): + for key, value in data_dict.items(): new_prefix = prefix + [key] if isinstance(value, dict): for pair in _flatten_dict_nest_repeats(value, new_prefix): @@ -251,18 +281,17 @@ def _flatten_dict_nest_repeats(d, prefix): elif isinstance(value, list): repeats = [] - for i, item in enumerate(value): + for _i, item in enumerate(value): item_prefix = list(new_prefix) # make a copy if isinstance(item, dict): repeat = {} - for path, value in _flatten_dict_nest_repeats( - item, item_prefix): - # TODO: this only considers the first level of repeats - repeat.update({u"/".join(path[1:]): value}) + for path, r_value in _flatten_dict_nest_repeats(item, item_prefix): + # This only considers the first level of repeats + repeat.update({"/".join(path[1:]): r_value}) repeats.append(repeat) else: - repeats.append({u"/".join(item_prefix[1:]): item}) + repeats.append({"/".join(item_prefix[1:]): item}) yield (new_prefix, repeats) else: yield (new_prefix, value) @@ -281,6 +310,9 @@ def _gather_parent_node_list(node): def xpath_from_xml_node(node): + """ + Returns the xpath of an XML node. + """ node_names = _gather_parent_node_list(node) return "/".join(node_names[1:]) @@ -299,27 +331,37 @@ def _get_all_attributes(node): yield pair -class XFormInstanceParser(object): +class XFormInstanceParser: + """ + XFormInstanceParser - parses an XML string into an XML object. + """ def __init__(self, xml_str, data_dictionary): - self.dd = data_dictionary + # pylint: disable=invalid-name + self.data_dicionary = data_dictionary self.parse(xml_str) def parse(self, xml_str): + """ + Parses a submission XML into a python dictionary object. + """ self._xml_obj = clean_and_parse_xml(xml_str) self._root_node = self._xml_obj.documentElement - repeats = [e.get_abbreviated_xpath() - for e in self.dd.get_survey_elements_of_type(u"repeat")] - - self._dict = _xml_node_to_dict(self._root_node, repeats, - self.dd.encrypted) + repeats = [ + e.get_abbreviated_xpath() + for e in self.data_dicionary.get_survey_elements_of_type("repeat") + ] + + self._dict = _xml_node_to_dict( + self._root_node, repeats, self.data_dicionary.encrypted + ) self._flat_dict = {} if self._dict is None: raise InstanceEmptyError for path, value in _flatten_dict_nest_repeats(self._dict, []): - self._flat_dict[u"/".join(path[1:])] = value + self._flat_dict["/".join(path[1:])] = value self._set_attributes() def get_root_node(self): @@ -341,6 +383,7 @@ def get_attributes(self): return self._attributes def _set_attributes(self): + # pylint: disable=attribute-defined-outside-init self._attributes = {} all_attributes = list(_get_all_attributes(self._root_node)) for key, value in all_attributes: @@ -348,19 +391,29 @@ def _set_attributes(self): # multiple xml tags, overriding and log when this occurs if key in self._attributes: logger = logging.getLogger("console_logger") - logger.debug("Skipping duplicate attribute: %s" - " with value %s" % (key, value)) - logger.debug(text(all_attributes)) + logger.debug( + "Skipping duplicate attribute: %s" " with value %s", key, value + ) + logger.debug(str(all_attributes)) else: self._attributes[key] = value def get_xform_id_string(self): - return self._attributes[u"id"] + """ + Returns the submission XML `id` attribute. + """ + return self._attributes["id"] def get_version(self): - return self._attributes.get(u"version") + """ + Returns the submission XML version attribute. + """ + return self._attributes.get("version") def get_flat_dict_with_attributes(self): + """ + Adds the submission XML top level attributes to the resulting python object. + """ result = self.to_flat_dict().copy() result[XFORM_ID_STRING] = self.get_xform_id_string() @@ -372,15 +425,25 @@ def get_flat_dict_with_attributes(self): def xform_instance_to_dict(xml_str, data_dictionary): + """ + Parses an XForm submission XML into a python object. + """ parser = XFormInstanceParser(xml_str, data_dictionary) return parser.to_dict() def xform_instance_to_flat_dict(xml_str, data_dictionary): + """ + Parses an XForm submission XML into a flattened python object. + """ parser = XFormInstanceParser(xml_str, data_dictionary) return parser.to_flat_dict() def parse_xform_instance(xml_str, data_dictionary): + """ + Parses an XForm submission XML into a flattened python object + with additional attributes. + """ parser = XFormInstanceParser(xml_str, data_dictionary) return parser.get_flat_dict_with_attributes() diff --git a/onadata/apps/main/forms.py b/onadata/apps/main/forms.py index 1e3176f56c..7a4afd10ea 100644 --- a/onadata/apps/main/forms.py +++ b/onadata/apps/main/forms.py @@ -3,20 +3,20 @@ forms module. """ import os +import random import re -from future.moves.urllib.parse import urlparse -from future.moves.urllib.request import urlopen +from six.moves.urllib.parse import urlparse import requests from django import forms from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.files.base import ContentFile from django.core.files.storage import default_storage from django.core.validators import URLValidator from django.forms import ModelForm -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy from registration.forms import RegistrationFormUniqueEmail # pylint: disable=ungrouped-imports @@ -28,37 +28,42 @@ from onadata.libs.utils.user_auth import get_user_default_project FORM_LICENSES_CHOICES = ( - ('No License', ugettext_lazy('No License')), - ('https://creativecommons.org/licenses/by/3.0/', - ugettext_lazy('Attribution CC BY')), - ('https://creativecommons.org/licenses/by-sa/3.0/', - ugettext_lazy('Attribution-ShareAlike CC BY-SA')), + ("No License", gettext_lazy("No License")), + ( + "https://creativecommons.org/licenses/by/3.0/", + gettext_lazy("Attribution CC BY"), + ), + ( + "https://creativecommons.org/licenses/by-sa/3.0/", + gettext_lazy("Attribution-ShareAlike CC BY-SA"), + ), ) DATA_LICENSES_CHOICES = ( - ('No License', ugettext_lazy('No License')), - ('http://opendatacommons.org/licenses/pddl/summary/', - ugettext_lazy('PDDL')), - ('http://opendatacommons.org/licenses/by/summary/', - ugettext_lazy('ODC-BY')), - ('http://opendatacommons.org/licenses/odbl/summary/', - ugettext_lazy('ODBL')), + ("No License", gettext_lazy("No License")), + ("http://opendatacommons.org/licenses/pddl/summary/", gettext_lazy("PDDL")), + ("http://opendatacommons.org/licenses/by/summary/", gettext_lazy("ODC-BY")), + ("http://opendatacommons.org/licenses/odbl/summary/", gettext_lazy("ODBL")), ) PERM_CHOICES = ( - ('view', ugettext_lazy('Can view')), - ('edit', ugettext_lazy('Can edit')), - ('report', ugettext_lazy('Can submit to')), - ('remove', ugettext_lazy('Remove permissions')), + ("view", gettext_lazy("Can view")), + ("edit", gettext_lazy("Can edit")), + ("report", gettext_lazy("Can submit to")), + ("remove", gettext_lazy("Remove permissions")), ) VALID_XLSFORM_CONTENT_TYPES = [ - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - 'text/csv', - 'application/vnd.ms-excel' + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "text/csv", + "application/vnd.ms-excel", ] -VALID_FILE_EXTENSIONS = ['.xlsx', '.csv'] +VALID_FILE_EXTENSIONS = [".xlsx", ".csv"] + + +# pylint: disable=invalid-name +User = get_user_model() def get_filename(response): @@ -70,11 +75,11 @@ def get_filename(response): # following format: # 'attachment; filename="ActApp_Survey_System.xlsx"; filename*=UTF-8\'\'ActApp_Survey_System.xlsx' # noqa cleaned_xls_file = "" - content = response.headers.get('content-disposition').split('; ') - counter = [a for a in content if a.startswith('filename=')] - if len(counter) >= 1: + content = response.headers.get("Content-Disposition").split("; ") + counter = [a for a in content if a.startswith("filename=")] + if counter: filename_key_val = counter[0] - filename = filename_key_val.split('=')[1].replace("\"", "") + filename = filename_key_val.split("=")[1].replace('"', "") name, extension = os.path.splitext(filename) if extension in VALID_FILE_EXTENSIONS and name: @@ -84,42 +89,46 @@ def get_filename(response): class DataLicenseForm(forms.Form): - """" + """ " Data license form. """ - value = forms.ChoiceField(choices=DATA_LICENSES_CHOICES, - widget=forms.Select( - attrs={'disabled': 'disabled', - 'id': 'data-license'})) + + value = forms.ChoiceField( + choices=DATA_LICENSES_CHOICES, + widget=forms.Select(attrs={"disabled": "disabled", "id": "data-license"}), + ) class FormLicenseForm(forms.Form): """ Form license form. """ - value = forms.ChoiceField(choices=FORM_LICENSES_CHOICES, - widget=forms.Select( - attrs={'disabled': 'disabled', - 'id': 'form-license'})) + + value = forms.ChoiceField( + choices=FORM_LICENSES_CHOICES, + widget=forms.Select(attrs={"disabled": "disabled", "id": "form-license"}), + ) class PermissionForm(forms.Form): """ Permission assignment form. """ + for_user = forms.CharField( widget=forms.TextInput( attrs={ - 'id': 'autocomplete', - 'data-provide': 'typeahead', - 'autocomplete': 'off' - }) + "id": "autocomplete", + "data-provide": "typeahead", + "autocomplete": "off", + } + ) ) perm_type = forms.ChoiceField(choices=PERM_CHOICES, widget=forms.Select()) def __init__(self, username): self.username = username - super(PermissionForm, self).__init__() + super().__init__() class UserProfileForm(ModelForm): @@ -129,59 +138,67 @@ class UserProfileForm(ModelForm): class Meta: model = UserProfile - exclude = ('user', 'created_by', 'num_of_submissions') + # pylint: disable=modelform-uses-exclude + exclude = ("user", "created_by", "num_of_submissions") + email = forms.EmailField(widget=forms.TextInput()) def clean_metadata(self): """ Returns an empty dict if metadata is None. """ - metadata = self.cleaned_data.get('metadata') + metadata = self.cleaned_data.get("metadata") - return metadata if metadata is not None else dict() + return metadata if metadata is not None else {} class UserProfileFormRegister(forms.Form): """ User profile registration form. """ - first_name = forms.CharField(widget=forms.TextInput(), required=True, - max_length=255) - last_name = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) - city = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) - country = forms.ChoiceField(widget=forms.Select(), required=False, - choices=COUNTRIES, initial='ZZ') - organization = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) - home_page = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) - twitter = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) + + first_name = forms.CharField( + widget=forms.TextInput(), required=True, max_length=255 + ) + last_name = forms.CharField( + widget=forms.TextInput(), required=False, max_length=255 + ) + city = forms.CharField(widget=forms.TextInput(), required=False, max_length=255) + country = forms.ChoiceField( + widget=forms.Select(), required=False, choices=COUNTRIES, initial="ZZ" + ) + organization = forms.CharField( + widget=forms.TextInput(), required=False, max_length=255 + ) + home_page = forms.CharField( + widget=forms.TextInput(), required=False, max_length=255 + ) + twitter = forms.CharField(widget=forms.TextInput(), required=False, max_length=255) def save_user_profile(self, new_user): """ Creates and returns a new_user profile. """ - new_profile = \ - UserProfile(user=new_user, name=self.cleaned_data['first_name'], - city=self.cleaned_data['city'], - country=self.cleaned_data['country'], - organization=self.cleaned_data['organization'], - home_page=self.cleaned_data['home_page'], - twitter=self.cleaned_data['twitter']) + new_profile = UserProfile( + user=new_user, + name=self.cleaned_data["first_name"], + city=self.cleaned_data["city"], + country=self.cleaned_data["country"], + organization=self.cleaned_data["organization"], + home_page=self.cleaned_data["home_page"], + twitter=self.cleaned_data["twitter"], + ) new_profile.save() return new_profile # pylint: disable=too-many-ancestors # order of inheritance control order of form display -class RegistrationFormUserProfile(RegistrationFormUniqueEmail, - UserProfileFormRegister): +class RegistrationFormUserProfile(RegistrationFormUniqueEmail, UserProfileFormRegister): """ User profile registration form. """ + RESERVED_USERNAMES = settings.RESERVED_USERNAMES username = forms.CharField(widget=forms.TextInput(), max_length=30) email = forms.EmailField(widget=forms.TextInput()) @@ -191,133 +208,146 @@ def clean_username(self): """ Validate a new user username. """ - username = self.cleaned_data['username'].lower() + username = self.cleaned_data["username"].lower() if username in self.RESERVED_USERNAMES: raise forms.ValidationError( - _(u'%s is a reserved name, please choose another') % username) - elif not self.legal_usernames_re.search(username): + _(f"{username} is a reserved name, please choose another") + ) + if not self.legal_usernames_re.search(username): raise forms.ValidationError( - _(u'username may only contain alpha-numeric characters and ' - u'underscores')) + _("username may only contain alpha-numeric characters and underscores") + ) try: User.objects.get(username=username) except User.DoesNotExist: return username - raise forms.ValidationError(_(u'%s already exists') % username) + raise forms.ValidationError(_("%s already exists") % username) class SourceForm(forms.Form): """ Source document form. """ - source = forms.FileField(label=ugettext_lazy(u"Source document"), - required=True) + + source = forms.FileField(label=gettext_lazy("Source document"), required=True) class SupportDocForm(forms.Form): """ Supporting document. """ - doc = forms.FileField(label=ugettext_lazy(u"Supporting document"), - required=True) + + doc = forms.FileField(label=gettext_lazy("Supporting document"), required=True) class MediaForm(forms.Form): """ Media file upload form. """ - media = forms.FileField(label=ugettext_lazy(u"Media upload"), - required=True) + + media = forms.FileField(label=gettext_lazy("Media upload"), required=True) def clean_media(self): """ Validate media upload file. """ - data_type = self.cleaned_data['media'].content_type - if data_type not in ['image/jpeg', 'image/png', 'audio/mpeg']: - raise forms.ValidationError('Only these media types are \ - allowed .png .jpg .mp3 .3gp .wav') + data_type = self.cleaned_data["media"].content_type + if data_type not in ["image/jpeg", "image/png", "audio/mpeg"]: + raise forms.ValidationError( + "Only these media types are \ + allowed .png .jpg .mp3 .3gp .wav" + ) class MapboxLayerForm(forms.Form): """ Mapbox layers form. """ - map_name = forms.CharField(widget=forms.TextInput(), required=True, - max_length=255) - attribution = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) - link = forms.URLField(label=ugettext_lazy(u'JSONP url'), - required=True) + + map_name = forms.CharField(widget=forms.TextInput(), required=True, max_length=255) + attribution = forms.CharField( + widget=forms.TextInput(), required=False, max_length=255 + ) + link = forms.URLField(label=gettext_lazy("JSONP url"), required=True) class QuickConverterFile(forms.Form): """ Uploads XLSForm form. """ - xls_file = forms.FileField( - label=ugettext_lazy(u'XLS File'), required=False) + + xls_file = forms.FileField(label=gettext_lazy("XLS File"), required=False) class QuickConverterURL(forms.Form): """ Uploads XLSForm from a URL. """ - xls_url = forms.URLField(label=ugettext_lazy('XLS URL'), - required=False) + + xls_url = forms.URLField(label=gettext_lazy("XLS URL"), required=False) class QuickConverterDropboxURL(forms.Form): """ Uploads XLSForm from Dropbox. """ - dropbox_xls_url = forms.URLField( - label=ugettext_lazy('XLS URL'), required=False) + + dropbox_xls_url = forms.URLField(label=gettext_lazy("XLS URL"), required=False) class QuickConverterCsvFile(forms.Form): """ Uploads CSV XLSForm. """ - csv_url = forms.URLField( - label=ugettext_lazy('CSV URL'), required=False) + + csv_url = forms.URLField(label=gettext_lazy("CSV URL"), required=False) class QuickConverterTextXlsForm(forms.Form): """ Uploads Text XLSForm. """ + text_xls_form = forms.CharField( - label=ugettext_lazy('XLSForm Representation'), required=False) + label=gettext_lazy("XLSForm Representation"), required=False + ) class QuickConverterXmlFile(forms.Form): """ Uploads an XForm XML. """ - xml_file = forms.FileField( - label=ugettext_lazy(u'XML File'), required=False) + + xml_file = forms.FileField(label=gettext_lazy("XML File"), required=False) class QuickConverterFloipFile(forms.Form): """ Uploads a FLOIP results data package descriptor file. """ + floip_file = forms.FileField( - label=ugettext_lazy(u'FlOIP results data packages descriptor File'), - required=False) + label=gettext_lazy("FlOIP results data packages descriptor File"), + required=False, + ) # pylint: disable=too-many-ancestors -class QuickConverter(QuickConverterFile, QuickConverterURL, - QuickConverterDropboxURL, QuickConverterTextXlsForm, - QuickConverterCsvFile, QuickConverterXmlFile, - QuickConverterFloipFile): +class QuickConverter( + QuickConverterFile, + QuickConverterURL, + QuickConverterDropboxURL, + QuickConverterTextXlsForm, + QuickConverterCsvFile, + QuickConverterXmlFile, + QuickConverterFloipFile, +): """ Publish XLSForm and convert to XForm. """ + project = forms.IntegerField(required=False) validate = URLValidator() @@ -325,17 +355,17 @@ def clean_project(self): """ Project validation. """ - project = self.cleaned_data['project'] + project = self.cleaned_data["project"] if project is not None: try: - # pylint: disable=attribute-defined-outside-init, no-member + # pylint: disable=attribute-defined-outside-init,no-member self._project = Project.objects.get(pk=int(project)) - except (Project.DoesNotExist, ValueError): - raise forms.ValidationError( - _(u"Unknown project id: %s" % project)) + except (Project.DoesNotExist, ValueError) as e: + raise forms.ValidationError(_(f"Unknown project id: {project}")) from e return project + # pylint: disable=too-many-locals def publish(self, user, id_string=None, created_by=None): """ Publish XLSForm. @@ -345,95 +375,108 @@ def publish(self, user, id_string=None, created_by=None): # this will save the file and pass it instead of the 'xls_file' # field. cleaned_xls_file = None - if 'text_xls_form' in self.cleaned_data\ - and self.cleaned_data['text_xls_form'].strip(): - csv_data = self.cleaned_data['text_xls_form'] + if ( + "text_xls_form" in self.cleaned_data + and self.cleaned_data["text_xls_form"].strip() + ): + csv_data = self.cleaned_data["text_xls_form"] # assigning the filename to a random string (quick fix) - import random - rand_name = "uploaded_form_%s.csv" % ''.join( - random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6)) - - cleaned_xls_file = \ - default_storage.save( - upload_to(None, rand_name, user.username), - ContentFile(csv_data.encode())) - if 'xls_file' in self.cleaned_data and\ - self.cleaned_data['xls_file']: - cleaned_xls_file = self.cleaned_data['xls_file'] - if 'floip_file' in self.cleaned_data and\ - self.cleaned_data['floip_file']: - cleaned_xls_file = self.cleaned_data['floip_file'] + + random_string = "".join( + random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6) + ) + rand_name = f"uploaded_form_{random_string}.csv" + + cleaned_xls_file = default_storage.save( + upload_to(None, rand_name, user.username), + ContentFile(csv_data.encode()), + ) + if "xls_file" in self.cleaned_data and self.cleaned_data["xls_file"]: + cleaned_xls_file = self.cleaned_data["xls_file"] + if "floip_file" in self.cleaned_data and self.cleaned_data["floip_file"]: + cleaned_xls_file = self.cleaned_data["floip_file"] cleaned_url = ( - self.cleaned_data['xls_url'].strip() or - self.cleaned_data['dropbox_xls_url'] or - self.cleaned_data['csv_url']) + self.cleaned_data["xls_url"].strip() + or self.cleaned_data["dropbox_xls_url"] + or self.cleaned_data["csv_url"] + ) if cleaned_url: + self.validate(cleaned_url) cleaned_xls_file = urlparse(cleaned_url) - cleaned_xls_file = \ - '_'.join(cleaned_xls_file.path.split('/')[-2:]) + cleaned_xls_file = "_".join(cleaned_xls_file.path.split("/")[-2:]) name, extension = os.path.splitext(cleaned_xls_file) if extension not in VALID_FILE_EXTENSIONS and name: - response = requests.get(cleaned_url) - if response.headers.get('content-type') in \ - VALID_XLSFORM_CONTENT_TYPES and \ - response.status_code < 400: + response = requests.head(cleaned_url) + if ( + response.headers.get("content-type") + in VALID_XLSFORM_CONTENT_TYPES + and response.status_code < 400 + ): cleaned_xls_file = get_filename(response) - cleaned_xls_file = \ - upload_to(None, cleaned_xls_file, user.username) - self.validate(cleaned_url) - xls_data = ContentFile(urlopen(cleaned_url).read()) - cleaned_xls_file = \ - default_storage.save(cleaned_xls_file, xls_data) + cleaned_xls_file = upload_to(None, cleaned_xls_file, user.username) + response = requests.get(cleaned_url) + if response.status_code < 400: + xls_data = ContentFile(response.content) + cleaned_xls_file = default_storage.save(cleaned_xls_file, xls_data) - project = self.cleaned_data['project'] + project = self.cleaned_data["project"] if project is None: project = get_user_default_project(user) else: project = self._project - cleaned_xml_file = self.cleaned_data['xml_file'] + cleaned_xml_file = self.cleaned_data["xml_file"] if cleaned_xml_file: - return publish_xml_form(cleaned_xml_file, user, project, - id_string, created_by or user) + return publish_xml_form( + cleaned_xml_file, user, project, id_string, created_by or user + ) if cleaned_xls_file is None: raise forms.ValidationError( - _(u"XLSForm not provided, expecting either of these" - " params: 'xml_file', 'xls_file', 'xls_url', 'csv_url'," - " 'dropbox_xls_url', 'text_xls_form', 'floip_file'")) + _( + "XLSForm not provided, expecting either of these" + " params: 'xml_file', 'xls_file', 'xls_url', 'csv_url'," + " 'dropbox_xls_url', 'text_xls_form', 'floip_file'" + ) + ) # publish the xls - return publish_xls_form(cleaned_xls_file, user, project, - id_string, created_by or user) + return publish_xls_form( + cleaned_xls_file, user, project, id_string, created_by or user + ) + return None class ActivateSMSSupportForm(forms.Form): """ Enable SMS support form. """ - enable_sms_support = forms.TypedChoiceField(coerce=lambda x: x == 'True', - choices=((False, 'No'), - (True, 'Yes')), - widget=forms.Select, - label=ugettext_lazy( - u"Enable SMS Support")) - sms_id_string = forms.CharField(max_length=50, required=True, - label=ugettext_lazy(u"SMS Keyword")) + + enable_sms_support = forms.TypedChoiceField( + coerce=lambda x: x == "True", + choices=((False, "No"), (True, "Yes")), + widget=forms.Select, + label=gettext_lazy("Enable SMS Support"), + ) + sms_id_string = forms.CharField( + max_length=50, required=True, label=gettext_lazy("SMS Keyword") + ) def clean_sms_id_string(self): """ SMS id_string validation. """ - sms_id_string = self.cleaned_data.get('sms_id_string', '').strip() + sms_id_string = self.cleaned_data.get("sms_id_string", "").strip() - if not re.match(r'^[a-z0-9\_\-]+$', sms_id_string): - raise forms.ValidationError(u"id_string can only contain alphanum" - u" characters") + if not re.match(r"^[a-z0-9\_\-]+$", sms_id_string): + raise forms.ValidationError( + "id_string can only contain alphanum" " characters" + ) return sms_id_string @@ -442,8 +485,9 @@ class ExternalExportForm(forms.Form): """ XLS reports form. """ - template_name = forms.CharField(label='Template Name', max_length=20) - template_token = forms.URLField(label='Template URL', max_length=100) + + template_name = forms.CharField(label="Template Name", max_length=20) + template_token = forms.URLField(label="Template URL", max_length=100) # Deprecated diff --git a/onadata/apps/main/management/commands/create_enketo_express_urls.py b/onadata/apps/main/management/commands/create_enketo_express_urls.py index a926566901..c9658fbd0d 100644 --- a/onadata/apps/main/management/commands/create_enketo_express_urls.py +++ b/onadata/apps/main/management/commands/create_enketo_express_urls.py @@ -1,6 +1,6 @@ from django.core.management.base import BaseCommand, CommandError from django.http import HttpRequest -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models import XForm from onadata.libs.utils.model_tools import queryset_iterator @@ -9,7 +9,7 @@ class Command(BaseCommand): - help = ugettext_lazy("Create enketo url including preview") + help = gettext_lazy("Create enketo url including preview") def add_arguments(self, parser): parser.add_argument( diff --git a/onadata/apps/main/management/commands/create_metadata_for_kpi_deployed_forms.py b/onadata/apps/main/management/commands/create_metadata_for_kpi_deployed_forms.py index 9cea2a0914..c5e8105a29 100644 --- a/onadata/apps/main/management/commands/create_metadata_for_kpi_deployed_forms.py +++ b/onadata/apps/main/management/commands/create_metadata_for_kpi_deployed_forms.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from django.db import connection from onadata.apps.main.models import MetaData @@ -9,7 +9,7 @@ class Command(BaseCommand): - help = ugettext_lazy("Create metadata for kpi forms that are not editable") + help = gettext_lazy("Create metadata for kpi forms that are not editable") def handle(self, *args, **kwargs): cursor = connection.cursor() diff --git a/onadata/apps/main/management/commands/export_user_emails.py b/onadata/apps/main/management/commands/export_user_emails.py index 4be652abe1..42289e9067 100644 --- a/onadata/apps/main/management/commands/export_user_emails.py +++ b/onadata/apps/main/management/commands/export_user_emails.py @@ -1,14 +1,14 @@ #!/usr/bin/env python # vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.main.models import UserProfile from onadata.libs.utils.model_tools import queryset_iterator class Command(BaseCommand): - help = ugettext_lazy("Export users and emails") + help = gettext_lazy("Export users and emails") def handle(self, *args, **kwargs): self.stdout.write( diff --git a/onadata/apps/main/management/commands/get_accounts_with_duplicate_id_strings.py b/onadata/apps/main/management/commands/get_accounts_with_duplicate_id_strings.py index 657e34a83e..b13b8cdd2e 100644 --- a/onadata/apps/main/management/commands/get_accounts_with_duplicate_id_strings.py +++ b/onadata/apps/main/management/commands/get_accounts_with_duplicate_id_strings.py @@ -1,41 +1,58 @@ +# -*- coding=utf-8 -*- +""" +get_accounts_with_duplicate_id_strings - Retrieves accounts with duplicate id_strings +""" +from pprint import pprint + from django.core.management.base import BaseCommand -from onadata.apps.logger.models.xform import XForm -from django.utils.translation import ugettext_lazy from django.db.models import Count -from pprint import pprint +from django.utils.translation import gettext_lazy + +from onadata.apps.logger.models.xform import XForm class Command(BaseCommand): - help = ugettext_lazy("Retrieves accounts with duplicate id_strings") + """Retrieves accounts with duplicate id_strings""" + + help = gettext_lazy("Retrieves accounts with duplicate id_strings") + # pylint: disable=unused-argument def handle(self, *args, **kwargs): - duplicates = XForm.objects.values( - 'id_string', 'user__username').annotate( - id_string_count=Count('id_string')).filter(id_string_count__gt=1) + """Retrieves accounts with duplicate id_strings""" + duplicates = ( + XForm.objects.values("id_string", "user__username") + .annotate(id_string_count=Count("id_string")) + .filter(id_string_count__gt=1) + ) duplicates_dict = {} if len(duplicates) > 0: - for a in duplicates: + for dupe in duplicates: xforms = XForm.objects.filter( - id_string=a.get('id_string'), - user__username=a.get('user__username')) + id_string=dupe.get("id_string"), + user__username=dupe.get("user__username"), + ) for xform in xforms: if duplicates_dict.get(xform.user) is None: - duplicates_dict[xform.user] = [{ - "form id": xform.pk, - "no. of submission": xform.num_of_submissions, - "email": xform.user.email, - "id_string": xform.id_string - }] + duplicates_dict[xform.user] = [ + { + "form id": xform.pk, + "no. of submission": xform.num_of_submissions, + "email": xform.user.email, + "id_string": xform.id_string, + } + ] else: - duplicates_dict[xform.user].append({ - "form id": xform.pk, - "no. of submission": xform.num_of_submissions, - "email": xform.user.email, - "id_string": xform.id_string - }) + duplicates_dict[xform.user].append( + { + "form id": xform.pk, + "no. of submission": xform.num_of_submissions, + "email": xform.user.email, + "id_string": xform.id_string, + } + ) self.stdout.write( - 'Accounts with duplicate id_strings: %s' % - len(duplicates_dict)) + f"Accounts with duplicate id_strings: {len(duplicates_dict)}" + ) pprint(duplicates_dict) else: - self.stdout.write('Each account has a unique id_string :)') + self.stdout.write("Each account has a unique id_string :)") diff --git a/onadata/apps/main/management/commands/mailer.py b/onadata/apps/main/management/commands/mailer.py index af8406dbb2..014c07d1e3 100644 --- a/onadata/apps/main/management/commands/mailer.py +++ b/onadata/apps/main/management/commands/mailer.py @@ -1,13 +1,13 @@ from django.core.management.base import BaseCommand, CommandError from django.contrib.auth.models import User from django.template.loader import get_template -from django.utils.translation import ugettext as _, ugettext_lazy +from django.utils.translation import gettext as _, gettext_lazy from templated_email import send_templated_mail class Command(BaseCommand): - help = ugettext_lazy("Send an email to all onadata users") + help = gettext_lazy("Send an email to all onadata users") def add_arguments(self, parser): parser.add_argument("-m", "--message", dest="message", default=False) diff --git a/onadata/apps/main/management/commands/migrate_audit_log.py b/onadata/apps/main/management/commands/migrate_audit_log.py index ceacf5c51d..4abfd380eb 100644 --- a/onadata/apps/main/management/commands/migrate_audit_log.py +++ b/onadata/apps/main/management/commands/migrate_audit_log.py @@ -1,12 +1,12 @@ from django.conf import settings from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.main.models.audit import Audit class Command(BaseCommand): - help = ugettext_lazy("migrate audit log from mongo to postgres") + help = gettext_lazy("migrate audit log from mongo to postgres") def handle(self, *args, **kwargs): auditlog = settings.MONGO_DB.auditlog diff --git a/onadata/apps/main/management/commands/remove_odk_prefix.py b/onadata/apps/main/management/commands/remove_odk_prefix.py index 9d17e8b35e..093b51bff4 100644 --- a/onadata/apps/main/management/commands/remove_odk_prefix.py +++ b/onadata/apps/main/management/commands/remove_odk_prefix.py @@ -1,10 +1,10 @@ from django.core.management.base import BaseCommand from django.db import connection -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy class Command(BaseCommand): - help = ugettext_lazy("Remove from logger and viewer apps") + help = gettext_lazy("Remove from logger and viewer apps") option_list = BaseCommand.option_list diff --git a/onadata/apps/main/management/commands/set_media_file_hash.py b/onadata/apps/main/management/commands/set_media_file_hash.py index 44f12fba06..7b1d85077c 100644 --- a/onadata/apps/main/management/commands/set_media_file_hash.py +++ b/onadata/apps/main/management/commands/set_media_file_hash.py @@ -1,17 +1,21 @@ +# -*- coding: utf-8 -*- +"""set_media_file_hash command - (re)apply the hash of all media files.""" from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.main.models import MetaData from onadata.libs.utils.model_tools import queryset_iterator class Command(BaseCommand): - help = ugettext_lazy("Set media file_hash for all existing media files") + """Set media file_hash for all existing media files""" - option_list = BaseCommand.option_list + help = gettext_lazy("Set media file_hash for all existing media files") + # pylint: disable=unused-argument def handle(self, *args, **kwargs): - for media in queryset_iterator(MetaData.objects.exclude(data_file='')): + """Set media file_hash for all existing media files""" + for media in queryset_iterator(MetaData.objects.exclude(data_file="")): if media.data_file: - media.file_hash = media._set_hash() + media.file_hash = media.set_hash() media.save() diff --git a/onadata/apps/main/management/commands/update_enketo_urls.py b/onadata/apps/main/management/commands/update_enketo_urls.py index 96cc078c7a..89f18cb3c2 100644 --- a/onadata/apps/main/management/commands/update_enketo_urls.py +++ b/onadata/apps/main/management/commands/update_enketo_urls.py @@ -1,54 +1,73 @@ +# -*- coding: utf-8 -*- +""" +update_enketo_urls - command to update Enketo preview URLs in the MetaData model. +""" +import argparse +import sys + from django.core.management.base import BaseCommand, CommandError from django.db.models import Q from django.http import HttpRequest -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.main.models.meta_data import MetaData -from onadata.libs.utils.viewer_tools import ( - get_enketo_urls, get_form_url) +from onadata.libs.utils.viewer_tools import get_enketo_urls, get_form_url class Command(BaseCommand): - help = ugettext_lazy("Updates enketo preview urls in MetaData model") + """Updates enketo preview urls in MetaData model""" + + help = gettext_lazy("Updates enketo preview urls in MetaData model") def add_arguments(self, parser): parser.add_argument( - "-n", "--server_name", dest="server_name", default="enketo.ona.io") - parser.add_argument( - "-p", "--server_port", dest="server_port", default="443") + "-n", "--server_name", dest="server_name", default="enketo.ona.io" + ) + parser.add_argument("-p", "--server_port", dest="server_port", default="443") + parser.add_argument("-r", "--protocol", dest="protocol", default="https") parser.add_argument( - "-r", "--protocol", dest="protocol", default="https") + "-c", + "--generate_consistent_urls", + dest="generate_consistent_urls", + default=True, + ) parser.add_argument( - "-c", "--generate_consistent_urls", - dest="generate_consistent_urls", default=True) + "enketo_urls_file", argparse.FileType("w"), default=sys.stdout + ) + # pylint: disable=too-many-locals def handle(self, *args, **options): + """Updates enketo preview urls in MetaData model""" request = HttpRequest() - server_name = options.get('server_name') - server_port = options.get('server_port') - protocol = options.get('protocol') - generate_consistent_urls = options.get('generate_consistent_urls') + server_name = options.get("server_name") + server_port = options.get("server_port") + protocol = options.get("protocol") + generate_consistent_urls = options.get("generate_consistent_urls") + enketo_urls_file = options.get("enketo_urls_file") if not server_name or not server_port or not protocol: raise CommandError( - 'please provide a server_name, a server_port and a protocol') + "please provide a server_name, a server_port and a protocol" + ) - if server_name not in ['ona.io', 'stage.ona.io', 'localhost']: - raise CommandError('server name provided is not valid') + if server_name not in ["ona.io", "stage.ona.io", "localhost"]: + raise CommandError("server name provided is not valid") - if protocol not in ['http', 'https']: - raise CommandError('protocol provided is not valid') + if protocol not in ["http", "https"]: + raise CommandError("protocol provided is not valid") # required for generation of enketo url - request.META['HTTP_HOST'] = '%s:%s' % (server_name, server_port)\ - if server_port != '80' else server_name + request.META["HTTP_HOST"] = ( + f"{server_name}:{server_port}" if server_port != "80" else server_name + ) # required for generation of enketo preview url - request.META['SERVER_NAME'] = server_name - request.META['SERVER_PORT'] = server_port + request.META["SERVER_NAME"] = server_name + request.META["SERVER_PORT"] = server_port resultset = MetaData.objects.filter( - Q(data_type='enketo_url') | Q(data_type='enketo_preview_url')) + Q(data_type="enketo_url") | Q(data_type="enketo_preview_url") + ) for meta_data in resultset: username = meta_data.content_object.user.username @@ -59,21 +78,21 @@ def handle(self, *args, **options): xform = meta_data.content_object xform_pk = xform.pk - with open('/tmp/enketo_url', 'a') as f: - form_url = get_form_url( - request, username=username, id_string=id_string, - xform_pk=xform_pk, - generate_consistent_urls=generate_consistent_urls) - enketo_urls = get_enketo_urls(form_url, id_string) - if data_type == 'enketo_url': - _enketo_url = (enketo_urls.get('offline_url') or - enketo_urls.get('url')) - MetaData.enketo_url(xform, _enketo_url) - elif data_type == 'enketo_preview_url': - _enketo_preview_url = (enketo_urls.get('preview_url')) - MetaData.enketo_preview_url(xform, _enketo_preview_url) - - f.write('%s : %s \n' % (id_string, data_value)) - self.stdout.write('%s: %s' % (data_type, meta_data.data_value)) + form_url = get_form_url( + request, + username=username, + xform_pk=xform_pk, + generate_consistent_urls=generate_consistent_urls, + ) + enketo_urls = get_enketo_urls(form_url, id_string) + if data_type == "enketo_url": + _enketo_url = enketo_urls.get("offline_url") or enketo_urls.get("url") + MetaData.enketo_url(xform, _enketo_url) + elif data_type == "enketo_preview_url": + _enketo_preview_url = enketo_urls.get("preview_url") + MetaData.enketo_preview_url(xform, _enketo_preview_url) + + enketo_urls_file.write(f"{id_string} : {data_value} \n") + self.stdout.write(f"{data_type}: {meta_data.data_value}") self.stdout.write("enketo urls update complete!!") diff --git a/onadata/apps/main/migrations/0001_initial.py b/onadata/apps/main/migrations/0001_initial.py index 6ebb6d0d76..897498e945 100644 --- a/onadata/apps/main/migrations/0001_initial.py +++ b/onadata/apps/main/migrations/0001_initial.py @@ -3,7 +3,6 @@ from django.db import models, migrations import onadata.apps.main.models.meta_data -import jsonfield.fields import django.utils.timezone from django.conf import settings @@ -11,249 +10,401 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0001_initial'), - ('auth', '0001_initial'), + ("logger", "0001_initial"), + ("auth", "0001_initial"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ migrations.CreateModel( - name='Audit', + name="Audit", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('json', jsonfield.fields.JSONField()), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("json", models.JSONField()), ], - options={ - }, + options={}, bases=(models.Model,), ), migrations.CreateModel( - name='MetaData', + name="MetaData", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('data_type', models.CharField(max_length=255)), - ('data_value', models.CharField(max_length=255)), - ('data_file', - models.FileField( - null=True, - upload_to=onadata.apps.main.models.meta_data.upload_to, - blank=True)), - ('data_file_type', - models.CharField(max_length=255, null=True, blank=True)), - ('file_hash', - models.CharField(max_length=50, null=True, blank=True)), - ('xform', models.ForeignKey(to='logger.XForm', - on_delete=models.CASCADE)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("data_type", models.CharField(max_length=255)), + ("data_value", models.CharField(max_length=255)), + ( + "data_file", + models.FileField( + null=True, + upload_to=onadata.apps.main.models.meta_data.upload_to, + blank=True, + ), + ), + ( + "data_file_type", + models.CharField(max_length=255, null=True, blank=True), + ), + ("file_hash", models.CharField(max_length=50, null=True, blank=True)), + ( + "xform", + models.ForeignKey(to="logger.XForm", on_delete=models.CASCADE), + ), ], - options={ - }, + options={}, bases=(models.Model,), ), migrations.CreateModel( - name='TokenStorageModel', + name="TokenStorageModel", fields=[ - ('id', - models.ForeignKey( - related_name='google_id', primary_key=True, - serialize=False, to=settings.AUTH_USER_MODEL, - on_delete=models.CASCADE - )), - ('token', models.TextField()), + ( + "id", + models.ForeignKey( + related_name="google_id", + primary_key=True, + serialize=False, + to=settings.AUTH_USER_MODEL, + on_delete=models.CASCADE, + ), + ), + ("token", models.TextField()), ], - options={ - }, + options={}, bases=(models.Model,), ), migrations.CreateModel( - name='UserProfile', + name="UserProfile", fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('name', models.CharField(max_length=255, blank=True)), - ('city', models.CharField(max_length=255, blank=True)), - ('country', models.CharField( - blank=True, max_length=2, - choices=[ - (b'AF', 'Afghanistan'), (b'AL', 'Albania'), - (b'DZ', 'Algeria'), (b'AS', 'American Samoa'), - (b'AD', 'Andorra'), (b'AO', 'Angola'), - (b'AI', 'Anguilla'), (b'AQ', 'Antarctica'), - (b'AG', 'Antigua and Barbuda'), (b'AR', 'Argentina'), - (b'AM', 'Armenia'), (b'AW', 'Aruba'), - (b'AU', 'Australia'), (b'AT', 'Austria'), - (b'AZ', 'Azerbaijan'), (b'BS', 'Bahamas'), - (b'BH', 'Bahrain'), (b'BD', 'Bangladesh'), - (b'BB', 'Barbados'), (b'BY', 'Belarus'), - (b'BE', 'Belgium'), (b'BZ', 'Belize'), - (b'BJ', 'Benin'), (b'BM', 'Bermuda'), - (b'BT', 'Bhutan'), (b'BO', 'Bolivia'), - (b'BQ', 'Bonaire, Sint Eustatius and Saba'), - (b'BA', 'Bosnia and Herzegovina'), (b'BW', 'Botswana'), - (b'BR', 'Brazil'), - (b'IO', 'British Indian Ocean Territory'), - (b'BN', 'Brunei Darussalam'), (b'BG', 'Bulgaria'), - (b'BF', 'Burkina Faso'), (b'BI', 'Burundi'), - (b'KH', 'Cambodia'), (b'CM', 'Cameroon'), - (b'CA', 'Canada'), (b'CV', 'Cape Verde'), - (b'KY', 'Cayman Islands'), - (b'CF', 'Central African Republic'), (b'TD', 'Chad'), - (b'CL', 'Chile'), (b'CN', 'China'), - (b'CX', 'Christmas Island'), - (b'CC', 'Cocos (Keeling) Islands'), - (b'CO', 'Colombia'), (b'KM', 'Comoros'), - (b'CG', 'Congo'), - (b'CD', 'Congo, The Democratic Republic of the'), - (b'CK', 'Cook Islands'), (b'CR', 'Costa Rica'), - (b'CI', 'Ivory Coast'), (b'HR', 'Croatia'), - (b'CU', 'Cuba'), (b'CW', 'Curacao'), (b'CY', 'Cyprus'), - (b'CZ', 'Czech Republic'), (b'DK', 'Denmark'), - (b'DJ', 'Djibouti'), (b'DM', 'Dominica'), - (b'DO', 'Dominican Republic'), (b'EC', 'Ecuador'), - (b'EG', 'Egypt'), (b'SV', 'El Salvador'), - (b'GQ', 'Equatorial Guinea'), (b'ER', 'Eritrea'), - (b'EE', 'Estonia'), (b'ET', 'Ethiopia'), - (b'FK', 'Falkland Islands (Malvinas)'), - (b'FO', 'Faroe Islands'), (b'FJ', 'Fiji'), - (b'FI', 'Finland'), (b'FR', 'France'), - (b'GF', 'French Guiana'), (b'PF', 'French Polynesia'), - (b'TF', 'French Southern Territories'), - (b'GA', 'Gabon'), (b'GM', 'Gambia'), - (b'GE', 'Georgia'), (b'DE', 'Germany'), - (b'GH', 'Ghana'), (b'GI', 'Gibraltar'), - (b'GR', 'Greece'), (b'GL', 'Greenland'), - (b'GD', 'Grenada'), (b'GP', 'Guadeloupe'), - (b'GU', 'Guam'), (b'GT', 'Guatemala'), - (b'GG', 'Guernsey'), (b'GN', 'Guinea'), - (b'GW', 'Guinea-Bissau'), (b'GY', 'Guyana'), - (b'HT', 'Haiti'), - (b'HM', 'Heard Island and McDonald Islands'), - (b'VA', 'Holy See (Vatican City State)'), - (b'HN', 'Honduras'), (b'HK', 'Hong Kong'), - (b'HU', 'Hungary'), (b'IS', 'Iceland'), - (b'IN', 'India'), (b'ID', 'Indonesia'), - (b'XZ', 'Installations in International Waters'), - (b'IR', 'Iran, Islamic Republic of'), (b'IQ', 'Iraq'), - (b'IE', 'Ireland'), (b'IM', 'Isle of Man'), - (b'IL', 'Israel'), (b'IT', 'Italy'), - (b'JM', 'Jamaica'), (b'JP', 'Japan'), - (b'JE', 'Jersey'), (b'JO', 'Jordan'), - (b'KZ', 'Kazakhstan'), (b'KE', 'Kenya'), - (b'KI', 'Kiribati'), - (b'KP', "Korea, Democratic People's Republic of"), - (b'KR', 'Korea, Republic of'), (b'XK', 'Kosovo'), - (b'KW', 'Kuwait'), (b'KG', 'Kyrgyzstan'), - (b'LA', "Lao People's Democratic Republic"), - (b'LV', 'Latvia'), (b'LB', 'Lebanon'), - (b'LS', 'Lesotho'), (b'LR', 'Liberia'), - (b'LY', 'Libyan Arab Jamahiriya'), - (b'LI', 'Liechtenstein'), (b'LT', 'Lithuania'), - (b'LU', 'Luxembourg'), (b'MO', 'Macao'), - (b'MK', 'Macedonia, The former Yugoslav Republic of'), - (b'MG', 'Madagascar'), (b'MW', 'Malawi'), - (b'MY', 'Malaysia'), (b'MV', 'Maldives'), - (b'ML', 'Mali'), (b'MT', 'Malta'), - (b'MH', 'Marshall Islands'), (b'MQ', 'Martinique'), - (b'MR', 'Mauritania'), (b'MU', 'Mauritius'), - (b'YT', 'Mayotte'), (b'MX', 'Mexico'), - (b'FM', 'Micronesia, Federated States of'), - (b'MD', 'Moldova, Republic of'), (b'MC', 'Monaco'), - (b'MN', 'Mongolia'), (b'ME', 'Montenegro'), - (b'MS', 'Montserrat'), (b'MA', 'Morocco'), - (b'MZ', 'Mozambique'), (b'MM', 'Myanmar'), - (b'NA', 'Namibia'), (b'NR', 'Nauru'), (b'NP', 'Nepal'), - (b'NL', 'Netherlands'), (b'NC', 'New Caledonia'), - (b'NZ', 'New Zealand'), (b'NI', 'Nicaragua'), - (b'NE', 'Niger'), (b'NG', 'Nigeria'), (b'NU', 'Niue'), - (b'NF', 'Norfolk Island'), - (b'MP', 'Northern Mariana Islands'), (b'NO', 'Norway'), - (b'OM', 'Oman'), (b'PK', 'Pakistan'), (b'PW', 'Palau'), - (b'PS', 'Palestinian Territory, Occupied'), - (b'PA', 'Panama'), (b'PG', 'Papua New Guinea'), - (b'PY', 'Paraguay'), (b'PE', 'Peru'), - (b'PH', 'Philippines'), (b'PN', 'Pitcairn'), - (b'PL', 'Poland'), (b'PT', 'Portugal'), - (b'PR', 'Puerto Rico'), (b'QA', 'Qatar'), - (b'RE', 'Reunion'), (b'RO', 'Romania'), - (b'RU', 'Russian Federation'), (b'RW', 'Rwanda'), - (b'SH', 'Saint Helena'), - (b'KN', 'Saint Kitts and Nevis'), - (b'LC', 'Saint Lucia'), - (b'PM', 'Saint Pierre and Miquelon'), - (b'VC', 'Saint Vincent and the Grenadines'), - (b'WS', 'Samoa'), (b'SM', 'San Marino'), - (b'ST', 'Sao Tome and Principe'), - (b'SA', 'Saudi Arabia'), (b'SN', 'Senegal'), - (b'RS', 'Serbia'), (b'SC', 'Seychelles'), - (b'SL', 'Sierra Leone'), (b'SG', 'Singapore'), - (b'SX', 'Sint Maarten (Dutch Part)'), - (b'SK', 'Slovakia'), (b'SI', 'Slovenia'), - (b'SB', 'Solomon Islands'), (b'SO', 'Somalia'), - (b'ZA', 'South Africa'), - (b'GS', - 'South Georgia and the South Sandwich Islands'), - (b'SS', 'South Sudan'), (b'ES', 'Spain'), - (b'LK', 'Sri Lanka'), (b'SD', 'Sudan'), - (b'SR', 'Suriname'), - (b'SJ', 'Svalbard and Jan Mayen'), - (b'SZ', 'Swaziland'), (b'SE', 'Sweden'), - (b'CH', 'Switzerland'), - (b'SY', 'Syrian Arab Republic'), - (b'TW', 'Taiwan, Province of China'), - (b'TJ', 'Tajikistan'), - (b'TZ', 'Tanzania, United Republic of'), - (b'TH', 'Thailand'), (b'TL', 'Timor-Leste'), - (b'TG', 'Togo'), (b'TK', 'Tokelau'), (b'TO', 'Tonga'), - (b'TT', 'Trinidad and Tobago'), (b'TN', 'Tunisia'), - (b'TR', 'Turkey'), (b'TM', 'Turkmenistan'), - (b'TC', 'Turks and Caicos Islands'), (b'TV', 'Tuvalu'), - (b'UG', 'Uganda'), (b'UA', 'Ukraine'), - (b'AE', 'United Arab Emirates'), - (b'GB', 'United Kingdom'), (b'US', 'United States'), - (b'UM', 'United States Minor Outlying Islands'), - (b'UY', 'Uruguay'), (b'UZ', 'Uzbekistan'), - (b'VU', 'Vanuatu'), (b'VE', 'Venezuela'), - (b'VN', 'Viet Nam'), - (b'VG', 'Virgin Islands, British'), - (b'VI', 'Virgin Islands, U.S.'), - (b'WF', 'Wallis and Futuna'), - (b'EH', 'Western Sahara'), (b'YE', 'Yemen'), - (b'ZM', 'Zambia'), (b'ZW', 'Zimbabwe'), - (b'ZZ', 'Unknown or unspecified country')])), - ('organization', models.CharField(max_length=255, blank=True)), - ('home_page', models.CharField(max_length=255, blank=True)), - ('twitter', models.CharField(max_length=255, blank=True)), - ('description', models.CharField(max_length=255, blank=True)), - ('require_auth', models.BooleanField( - default=False, - verbose_name='Require Phone Authentication')), - ('address', models.CharField(max_length=255, blank=True)), - ('phonenumber', models.CharField(max_length=30, blank=True)), - ('num_of_submissions', models.IntegerField(default=0)), - ('metadata', jsonfield.fields.JSONField(default={}, - blank=True)), - ('date_modified', models.DateTimeField( - default=django.utils.timezone.now, auto_now=True)), - ('created_by', models.ForeignKey( - blank=True, to=settings.AUTH_USER_MODEL, null=True, - on_delete=models.SET_NULL)), - ('user', models.OneToOneField(related_name='profile', - to=settings.AUTH_USER_MODEL, - on_delete=models.CASCADE)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("name", models.CharField(max_length=255, blank=True)), + ("city", models.CharField(max_length=255, blank=True)), + ( + "country", + models.CharField( + blank=True, + max_length=2, + choices=[ + (b"AF", "Afghanistan"), + (b"AL", "Albania"), + (b"DZ", "Algeria"), + (b"AS", "American Samoa"), + (b"AD", "Andorra"), + (b"AO", "Angola"), + (b"AI", "Anguilla"), + (b"AQ", "Antarctica"), + (b"AG", "Antigua and Barbuda"), + (b"AR", "Argentina"), + (b"AM", "Armenia"), + (b"AW", "Aruba"), + (b"AU", "Australia"), + (b"AT", "Austria"), + (b"AZ", "Azerbaijan"), + (b"BS", "Bahamas"), + (b"BH", "Bahrain"), + (b"BD", "Bangladesh"), + (b"BB", "Barbados"), + (b"BY", "Belarus"), + (b"BE", "Belgium"), + (b"BZ", "Belize"), + (b"BJ", "Benin"), + (b"BM", "Bermuda"), + (b"BT", "Bhutan"), + (b"BO", "Bolivia"), + (b"BQ", "Bonaire, Sint Eustatius and Saba"), + (b"BA", "Bosnia and Herzegovina"), + (b"BW", "Botswana"), + (b"BR", "Brazil"), + (b"IO", "British Indian Ocean Territory"), + (b"BN", "Brunei Darussalam"), + (b"BG", "Bulgaria"), + (b"BF", "Burkina Faso"), + (b"BI", "Burundi"), + (b"KH", "Cambodia"), + (b"CM", "Cameroon"), + (b"CA", "Canada"), + (b"CV", "Cape Verde"), + (b"KY", "Cayman Islands"), + (b"CF", "Central African Republic"), + (b"TD", "Chad"), + (b"CL", "Chile"), + (b"CN", "China"), + (b"CX", "Christmas Island"), + (b"CC", "Cocos (Keeling) Islands"), + (b"CO", "Colombia"), + (b"KM", "Comoros"), + (b"CG", "Congo"), + (b"CD", "Congo, The Democratic Republic of the"), + (b"CK", "Cook Islands"), + (b"CR", "Costa Rica"), + (b"CI", "Ivory Coast"), + (b"HR", "Croatia"), + (b"CU", "Cuba"), + (b"CW", "Curacao"), + (b"CY", "Cyprus"), + (b"CZ", "Czech Republic"), + (b"DK", "Denmark"), + (b"DJ", "Djibouti"), + (b"DM", "Dominica"), + (b"DO", "Dominican Republic"), + (b"EC", "Ecuador"), + (b"EG", "Egypt"), + (b"SV", "El Salvador"), + (b"GQ", "Equatorial Guinea"), + (b"ER", "Eritrea"), + (b"EE", "Estonia"), + (b"ET", "Ethiopia"), + (b"FK", "Falkland Islands (Malvinas)"), + (b"FO", "Faroe Islands"), + (b"FJ", "Fiji"), + (b"FI", "Finland"), + (b"FR", "France"), + (b"GF", "French Guiana"), + (b"PF", "French Polynesia"), + (b"TF", "French Southern Territories"), + (b"GA", "Gabon"), + (b"GM", "Gambia"), + (b"GE", "Georgia"), + (b"DE", "Germany"), + (b"GH", "Ghana"), + (b"GI", "Gibraltar"), + (b"GR", "Greece"), + (b"GL", "Greenland"), + (b"GD", "Grenada"), + (b"GP", "Guadeloupe"), + (b"GU", "Guam"), + (b"GT", "Guatemala"), + (b"GG", "Guernsey"), + (b"GN", "Guinea"), + (b"GW", "Guinea-Bissau"), + (b"GY", "Guyana"), + (b"HT", "Haiti"), + (b"HM", "Heard Island and McDonald Islands"), + (b"VA", "Holy See (Vatican City State)"), + (b"HN", "Honduras"), + (b"HK", "Hong Kong"), + (b"HU", "Hungary"), + (b"IS", "Iceland"), + (b"IN", "India"), + (b"ID", "Indonesia"), + (b"XZ", "Installations in International Waters"), + (b"IR", "Iran, Islamic Republic of"), + (b"IQ", "Iraq"), + (b"IE", "Ireland"), + (b"IM", "Isle of Man"), + (b"IL", "Israel"), + (b"IT", "Italy"), + (b"JM", "Jamaica"), + (b"JP", "Japan"), + (b"JE", "Jersey"), + (b"JO", "Jordan"), + (b"KZ", "Kazakhstan"), + (b"KE", "Kenya"), + (b"KI", "Kiribati"), + (b"KP", "Korea, Democratic People's Republic of"), + (b"KR", "Korea, Republic of"), + (b"XK", "Kosovo"), + (b"KW", "Kuwait"), + (b"KG", "Kyrgyzstan"), + (b"LA", "Lao People's Democratic Republic"), + (b"LV", "Latvia"), + (b"LB", "Lebanon"), + (b"LS", "Lesotho"), + (b"LR", "Liberia"), + (b"LY", "Libyan Arab Jamahiriya"), + (b"LI", "Liechtenstein"), + (b"LT", "Lithuania"), + (b"LU", "Luxembourg"), + (b"MO", "Macao"), + (b"MK", "Macedonia, The former Yugoslav Republic of"), + (b"MG", "Madagascar"), + (b"MW", "Malawi"), + (b"MY", "Malaysia"), + (b"MV", "Maldives"), + (b"ML", "Mali"), + (b"MT", "Malta"), + (b"MH", "Marshall Islands"), + (b"MQ", "Martinique"), + (b"MR", "Mauritania"), + (b"MU", "Mauritius"), + (b"YT", "Mayotte"), + (b"MX", "Mexico"), + (b"FM", "Micronesia, Federated States of"), + (b"MD", "Moldova, Republic of"), + (b"MC", "Monaco"), + (b"MN", "Mongolia"), + (b"ME", "Montenegro"), + (b"MS", "Montserrat"), + (b"MA", "Morocco"), + (b"MZ", "Mozambique"), + (b"MM", "Myanmar"), + (b"NA", "Namibia"), + (b"NR", "Nauru"), + (b"NP", "Nepal"), + (b"NL", "Netherlands"), + (b"NC", "New Caledonia"), + (b"NZ", "New Zealand"), + (b"NI", "Nicaragua"), + (b"NE", "Niger"), + (b"NG", "Nigeria"), + (b"NU", "Niue"), + (b"NF", "Norfolk Island"), + (b"MP", "Northern Mariana Islands"), + (b"NO", "Norway"), + (b"OM", "Oman"), + (b"PK", "Pakistan"), + (b"PW", "Palau"), + (b"PS", "Palestinian Territory, Occupied"), + (b"PA", "Panama"), + (b"PG", "Papua New Guinea"), + (b"PY", "Paraguay"), + (b"PE", "Peru"), + (b"PH", "Philippines"), + (b"PN", "Pitcairn"), + (b"PL", "Poland"), + (b"PT", "Portugal"), + (b"PR", "Puerto Rico"), + (b"QA", "Qatar"), + (b"RE", "Reunion"), + (b"RO", "Romania"), + (b"RU", "Russian Federation"), + (b"RW", "Rwanda"), + (b"SH", "Saint Helena"), + (b"KN", "Saint Kitts and Nevis"), + (b"LC", "Saint Lucia"), + (b"PM", "Saint Pierre and Miquelon"), + (b"VC", "Saint Vincent and the Grenadines"), + (b"WS", "Samoa"), + (b"SM", "San Marino"), + (b"ST", "Sao Tome and Principe"), + (b"SA", "Saudi Arabia"), + (b"SN", "Senegal"), + (b"RS", "Serbia"), + (b"SC", "Seychelles"), + (b"SL", "Sierra Leone"), + (b"SG", "Singapore"), + (b"SX", "Sint Maarten (Dutch Part)"), + (b"SK", "Slovakia"), + (b"SI", "Slovenia"), + (b"SB", "Solomon Islands"), + (b"SO", "Somalia"), + (b"ZA", "South Africa"), + (b"GS", "South Georgia and the South Sandwich Islands"), + (b"SS", "South Sudan"), + (b"ES", "Spain"), + (b"LK", "Sri Lanka"), + (b"SD", "Sudan"), + (b"SR", "Suriname"), + (b"SJ", "Svalbard and Jan Mayen"), + (b"SZ", "Swaziland"), + (b"SE", "Sweden"), + (b"CH", "Switzerland"), + (b"SY", "Syrian Arab Republic"), + (b"TW", "Taiwan, Province of China"), + (b"TJ", "Tajikistan"), + (b"TZ", "Tanzania, United Republic of"), + (b"TH", "Thailand"), + (b"TL", "Timor-Leste"), + (b"TG", "Togo"), + (b"TK", "Tokelau"), + (b"TO", "Tonga"), + (b"TT", "Trinidad and Tobago"), + (b"TN", "Tunisia"), + (b"TR", "Turkey"), + (b"TM", "Turkmenistan"), + (b"TC", "Turks and Caicos Islands"), + (b"TV", "Tuvalu"), + (b"UG", "Uganda"), + (b"UA", "Ukraine"), + (b"AE", "United Arab Emirates"), + (b"GB", "United Kingdom"), + (b"US", "United States"), + (b"UM", "United States Minor Outlying Islands"), + (b"UY", "Uruguay"), + (b"UZ", "Uzbekistan"), + (b"VU", "Vanuatu"), + (b"VE", "Venezuela"), + (b"VN", "Viet Nam"), + (b"VG", "Virgin Islands, British"), + (b"VI", "Virgin Islands, U.S."), + (b"WF", "Wallis and Futuna"), + (b"EH", "Western Sahara"), + (b"YE", "Yemen"), + (b"ZM", "Zambia"), + (b"ZW", "Zimbabwe"), + (b"ZZ", "Unknown or unspecified country"), + ], + ), + ), + ("organization", models.CharField(max_length=255, blank=True)), + ("home_page", models.CharField(max_length=255, blank=True)), + ("twitter", models.CharField(max_length=255, blank=True)), + ("description", models.CharField(max_length=255, blank=True)), + ( + "require_auth", + models.BooleanField( + default=False, verbose_name="Require Phone Authentication" + ), + ), + ("address", models.CharField(max_length=255, blank=True)), + ("phonenumber", models.CharField(max_length=30, blank=True)), + ("num_of_submissions", models.IntegerField(default=0)), + ("metadata", models.JSONField(default={}, blank=True)), + ( + "date_modified", + models.DateTimeField( + default=django.utils.timezone.now, auto_now=True + ), + ), + ( + "created_by", + models.ForeignKey( + blank=True, + to=settings.AUTH_USER_MODEL, + null=True, + on_delete=models.SET_NULL, + ), + ), + ( + "user", + models.OneToOneField( + related_name="profile", + to=settings.AUTH_USER_MODEL, + on_delete=models.CASCADE, + ), + ), ], options={ - 'permissions': ( - ('can_add_xform', - 'Can add/upload an xform to user profile'), - ('view_profile', 'Can view user profile')), + "permissions": ( + ("can_add_xform", "Can add/upload an xform to user profile"), + ("view_profile", "Can view user profile"), + ), }, bases=(models.Model,), ), migrations.AlterUniqueTogether( - name='metadata', - unique_together=set([('xform', 'data_type', 'data_value')]), + name="metadata", + unique_together=set([("xform", "data_type", "data_value")]), ), ] diff --git a/onadata/apps/main/migrations/0007_auto_20160418_0525.py b/onadata/apps/main/migrations/0007_auto_20160418_0525.py index d2f4dc5311..5778929ea9 100644 --- a/onadata/apps/main/migrations/0007_auto_20160418_0525.py +++ b/onadata/apps/main/migrations/0007_auto_20160418_0525.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals from django.db import migrations -import oauth2client.contrib.django_util.models +from onadata.apps.main.models.google_oath import CredentialsField class Migration(migrations.Migration): @@ -20,7 +20,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='tokenstoragemodel', name='credential', - field=oauth2client.contrib.django_util.models.CredentialsField( - null=True), + field=CredentialsField(null=True), ), ] diff --git a/onadata/apps/main/migrations/0010_auto_20220425_0313.py b/onadata/apps/main/migrations/0010_auto_20220425_0313.py new file mode 100644 index 0000000000..404cfa67a9 --- /dev/null +++ b/onadata/apps/main/migrations/0010_auto_20220425_0313.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.13 on 2022-04-25 07:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0009_auto_20190125_0517'), + ] + + operations = [ + migrations.AlterField( + model_name='audit', + name='json', + field=models.JSONField(), + ), + migrations.AlterField( + model_name='userprofile', + name='metadata', + field=models.JSONField(blank=True, default=dict), + ), + ] diff --git a/onadata/apps/main/migrations/0011_auto_20220510_0907.py b/onadata/apps/main/migrations/0011_auto_20220510_0907.py new file mode 100644 index 0000000000..1bcd5cc6da --- /dev/null +++ b/onadata/apps/main/migrations/0011_auto_20220510_0907.py @@ -0,0 +1,34 @@ +# Generated by Django 3.2.13 on 2022-05-10 13:07 +from django.db import migrations + +from google.oauth2.credentials import Credentials +from onadata.apps.main.models.google_oath import TokenStorageModel + + +def convert_oauth2client_credentials(apps, schema_editor): + """ + Converts the OAuth2Client Credentials class object + into a google.oauth2.credentials.Credentials object + """ + queryset = TokenStorageModel.objects.all() + for storage in queryset.iterator(): + if ( + isinstance(storage.credential, dict) + and "py/state" in storage.credential.keys() + ): + data = storage.credential.get("py/state") + scopes = data.pop("scopes") + if isinstance(scopes, dict): + scopes = scopes.get("py/set") + credential = Credentials.from_authorized_user_info(data, scopes=scopes) + storage.credential = credential + storage.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ("main", "0010_auto_20220425_0313"), + ] + + operations = [migrations.RunPython(convert_oauth2client_credentials)] diff --git a/onadata/apps/main/models/__init__.py b/onadata/apps/main/models/__init__.py index 8a4d74aff2..f342a92dc5 100644 --- a/onadata/apps/main/models/__init__.py +++ b/onadata/apps/main/models/__init__.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Main models. +""" from __future__ import absolute_import from onadata.apps.main.models.user_profile import UserProfile # noqa diff --git a/onadata/apps/main/models/audit.py b/onadata/apps/main/models/audit.py index 3a4d0ef4d2..8443b67e7a 100644 --- a/onadata/apps/main/models/audit.py +++ b/onadata/apps/main/models/audit.py @@ -1,107 +1,148 @@ +# -*- coding: utf-8 -*- +""" +Audit model +""" import json import six from django.db import models from django.db import connection -from django.contrib.postgres.fields import JSONField -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ DEFAULT_LIMIT = 1000 class Audit(models.Model): - json = JSONField() + """ + Audit model - persists audit logs. + """ + + # pylint: disable=no-member + json = models.JSONField() class Meta: - app_label = 'main' + app_label = "main" + +class AuditLog: + """ + AuditLog - creates and provide access to the Audit model records. + """ -class AuditLog(object): - ACCOUNT = u"account" + ACCOUNT = "account" DEFAULT_BATCHSIZE = 1000 - CREATED_ON = u"created_on" + CREATED_ON = "created_on" def __init__(self, data): self.data = data def save(self): - a = Audit(json=self.data) - a.save() + """ + Persists an audit to the DB + """ + audit = Audit(json=self.data) + audit.save() - return a + return audit @classmethod - def query_iterator(cls, sql, fields=None, params=[], count=False): + def query_iterator(cls, sql, fields=None, params=None, count=False): + """ + Returns an iterator of all records. + """ + # cursor seems to stringify dicts + # added workaround to parse stringified dicts to json + def parse_json(data): + """Helper function to return a JSON string ``data`` as a python object.""" + try: + return json.loads(data) + except ValueError: + return data + + params = params if params is not None else [] cursor = connection.cursor() sql_params = fields + params if fields is not None else params if count: - from_pos = sql.upper().find(' FROM') + from_pos = sql.upper().find(" FROM") if from_pos != -1: - sql = u"SELECT COUNT(*) " + sql[from_pos:] + sql = "SELECT COUNT(*) " + sql[from_pos:] - order_pos = sql.upper().find('ORDER BY') + order_pos = sql.upper().find("ORDER BY") if order_pos != -1: sql = sql[:order_pos] sql_params = params - fields = [u'count'] + fields = ["count"] cursor.execute(sql, sql_params) if fields is None: for row in cursor.fetchall(): - yield row[0] + yield parse_json(row[0]) else: for row in cursor.fetchall(): yield dict(zip(fields, row)) + # pylint: disable=too-many-locals,too-many-branches,too-many-arguments @classmethod - def query_data(cls, username, query=None, fields=None, sort=None, start=0, - limit=DEFAULT_LIMIT, count=False): + def query_data( + cls, + username, + query=None, + fields=None, + sort=None, + start=0, + limit=DEFAULT_LIMIT, + count=False, + ): + """ + Queries the Audit model and returns an iterator of the records. + """ if start is not None and (start < 0 or limit < 0): raise ValueError(_("Invalid start/limit params")) - sort = 'pk' if sort is None else sort - instances = Audit.objects.filter().extra(where=["json->>%s = %s"], - params=['account', username]) + sort = "pk" if sort is None else sort + instances = Audit.objects.filter().extra( + where=["json->>%s = %s"], params=["account", username] + ) where_params = [] - sql_where = u"" + sql_where = "" if query and isinstance(query, six.string_types): query = json.loads(query) or_where = [] or_params = [] - if '$or' in list(query): - or_dict = query.pop('$or') + if "$or" in list(query): + or_dict = query.pop("$or") for or_query in or_dict: - or_where.extend( - [u"json->>%s = %s" for i in or_query.items()]) + or_where.extend(["json->>%s = %s" for i in or_query.items()]) [ # pylint: disable=expression-not-assigned - or_params.extend(i) for i in or_query.items()] + or_params.extend(i) for i in or_query.items() + ] - or_where = [u"".join([u"(", u" OR ".join(or_where), u")"])] + or_where = ["".join(["(", " OR ".join(or_where), ")"])] - where = [u"json->>%s = %s" for i in query.items()] + or_where - [where_params.extend(i) for i in query.items()] + where = ["json->>%s = %s" for i in query.items()] + or_where + for i in query.items(): + where_params.extend(i) where_params.extend(or_params) if fields and isinstance(fields, six.string_types): fields = json.loads(fields) if fields: - field_list = [u"json->%s" for i in fields] - sql = u"SELECT %s FROM main_audit" % u",".join(field_list) + field_list = ["json->%s" for i in fields] + sql = f"SELECT {','.join(field_list)} FROM main_audit" if where_params: - sql_where = u" AND " + u" AND ".join(where) + sql_where = " AND " + " AND ".join(where) - sql += u" WHERE json->>%s = %s " + sql_where \ - + u" ORDER BY id" - params = ['account', username] + where_params + sql += " WHERE json->>%s = %s " + sql_where + " ORDER BY id" + params = ["account", username] + where_params if start is not None: - sql += u" OFFSET %s LIMIT %s" + sql += " OFFSET %s LIMIT %s" params += [start, limit] records = cls.query_iterator(sql, fields, params, count) else: @@ -111,21 +152,21 @@ def query_data(cls, username, query=None, fields=None, sort=None, start=0, if where_params: instances = instances.extra(where=where, params=where_params) - records = instances.values_list('json', flat=True) + records = instances.values_list("json", flat=True) sql, params = records.query.sql_with_params() - if isinstance(sort, six.string_types) and len(sort) > 0: - direction = 'DESC' if sort.startswith('-') else 'ASC' - sort = sort[1:] if sort.startswith('-') else sort - sql = u'{} ORDER BY json->>%s {}'.format(sql, direction) + if isinstance(sort, six.string_types) and sort: + direction = "DESC" if sort.startswith("-") else "ASC" + sort = sort[1:] if sort.startswith("-") else sort + sql = f"{sql} ORDER BY json->>%s {direction}" params += (sort,) if start is not None: # some inconsistent/weird behavior I noticed with django's # queryset made me have to do a raw query # records = records[start: limit] - sql = u"{} OFFSET %s LIMIT %s".format(sql) + sql = f"{sql} OFFSET %s LIMIT %s" params += (start, limit) records = cls.query_iterator(sql, None, list(params)) diff --git a/onadata/apps/main/models/google_oath.py b/onadata/apps/main/models/google_oath.py index 46df88fc01..b74b64cf37 100644 --- a/onadata/apps/main/models/google_oath.py +++ b/onadata/apps/main/models/google_oath.py @@ -2,9 +2,72 @@ """ Google auth token storage model class """ -from django.conf import settings +import base64 +import pickle + +from django.contrib.auth import get_user_model from django.db import models -from oauth2client.contrib.django_util.models import CredentialsField +from django.utils import encoding + +import jsonpickle +from google.oauth2.credentials import Credentials + + +class CredentialsField(models.Field): + """ + Django ORM field for storing OAuth2 Credentials. + Modified version of + https://github.com/onaio/oauth2client/blob/master/oauth2client/contrib/django_util/models.py + """ + + def __init__(self, *args, **kwargs): + if "null" not in kwargs: + kwargs["null"] = True + super().__init__(*args, **kwargs) + + def get_internal_type(self): + return "BinaryField" + + # pylint: disable=unused-argument + def from_db_value(self, value, expression, connection, context=None): + """Overrides ``models.Field`` method. This converts the value + returned from the database to an instance of this class. + """ + return self.to_python(value) + + def to_python(self, value): + """Overrides ``models.Field`` method. This is used to convert + bytes (from serialization etc) to an instance of this class""" + if value is None: + return None + if isinstance(value, Credentials): + return value + try: + return jsonpickle.decode( + base64.b64decode(encoding.smart_bytes(value)).decode() + ) + except ValueError: + return pickle.loads(base64.b64decode(encoding.smart_bytes(value))) + + def get_prep_value(self, value): + """Overrides ``models.Field`` method. This is used to convert + the value from an instances of this class to bytes that can be + inserted into the database. + """ + if value is None: + return None + return encoding.smart_str(base64.b64encode(jsonpickle.encode(value).encode())) + + def value_to_string(self, obj): + """Convert the field value from the provided model to a string. + Used during model serialization. + Args: + obj: db.Model, model object + Returns: + string, the serialized field value + """ + value = self.value_from_object(obj) + return self.get_prep_value(value) class TokenStorageModel(models.Model): @@ -12,10 +75,14 @@ class TokenStorageModel(models.Model): Google Auth Token storage model """ + # pylint: disable=invalid-name id = models.OneToOneField( - settings.AUTH_USER_MODEL, primary_key=True, related_name='google_id', - on_delete=models.CASCADE) + get_user_model(), + primary_key=True, + related_name="google_id", + on_delete=models.CASCADE, + ) credential = CredentialsField() class Meta: - app_label = 'main' + app_label = "main" diff --git a/onadata/apps/main/models/meta_data.py b/onadata/apps/main/models/meta_data.py index ecb3f599d1..025af48010 100644 --- a/onadata/apps/main/models/meta_data.py +++ b/onadata/apps/main/models/meta_data.py @@ -1,14 +1,16 @@ +# -*- coding: utf-8 -*- +""" +MetaData model +""" from __future__ import unicode_literals import logging import mimetypes import os from contextlib import closing -from hashlib import md5 +import hashlib -import requests from django.conf import settings -from django.utils import timezone from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ValidationError @@ -17,26 +19,31 @@ from django.core.validators import URLValidator from django.db import IntegrityError, models from django.db.models.signals import post_delete, post_save -from past.builtins import basestring +from django.utils import timezone + +import requests from onadata.libs.utils.cache_tools import XFORM_METADATA_CACHE, safe_delete -from onadata.libs.utils.common_tags import (GOOGLE_SHEET_DATA_TYPE, TEXTIT, - XFORM_META_PERMS, TEXTIT_DETAILS) +from onadata.libs.utils.common_tags import ( + GOOGLE_SHEET_DATA_TYPE, + TEXTIT, + TEXTIT_DETAILS, + XFORM_META_PERMS, +) +ANONYMOUS_USERNAME = "anonymous" CHUNK_SIZE = 1024 INSTANCE_MODEL_NAME = "instance" PROJECT_MODEL_NAME = "project" XFORM_MODEL_NAME = "xform" -urlvalidate = URLValidator() - -ANONYMOUS_USERNAME = "anonymous" - - def is_valid_url(uri): + """ + Validates a URI. + """ try: - urlvalidate(uri) + URLValidator()(uri) except ValidationError: return False @@ -44,59 +51,70 @@ def is_valid_url(uri): def upload_to(instance, filename): + """ + Returns the upload path for given ``filename``. + """ + is_instance_model = instance.content_type.model == INSTANCE_MODEL_NAME username = None - if instance.content_object.user is None and \ - instance.content_type.model == INSTANCE_MODEL_NAME: + if instance.content_object.user is None and is_instance_model: username = instance.content_object.xform.user.username else: username = instance.content_object.user.username - if instance.data_type == 'media': - return os.path.join(username, 'formid-media', filename) + if instance.data_type == "media": + return os.path.join(username, "formid-media", filename) - return os.path.join(username, 'docs', filename) + return os.path.join(username, "docs", filename) def save_metadata(metadata_obj): + """ + Saves the MetaData object and returns it. + """ try: metadata_obj.save() except IntegrityError: - logging.exception("MetaData object '%s' already exists" % metadata_obj) + logging.exception("MetaData object '%s' already exists", metadata_obj) return metadata_obj def get_default_content_type(): - content_object, created = ContentType.objects.get_or_create( - app_label="logger", model=XFORM_MODEL_NAME) + """ + Returns the default content type id for the XForm model. + """ + content_object, _created = ContentType.objects.get_or_create( + app_label="logger", model=XFORM_MODEL_NAME + ) return content_object.id -def unique_type_for_form(content_object, - data_type, - data_value=None, - data_file=None): +def unique_type_for_form(content_object, data_type, data_value=None, data_file=None): """ Ensure that each metadata object has unique xform and data_type fields return the metadata object """ - defaults = {'data_value': data_value} if data_value else {} + defaults = {"data_value": data_value} if data_value else {} content_type = ContentType.objects.get_for_model(content_object) if data_value is None and data_file is None: + # pylint: disable=no-member result = MetaData.objects.filter( - object_id=content_object.id, content_type=content_type, - data_type=data_type).first() + object_id=content_object.id, content_type=content_type, data_type=data_type + ).first() else: - result, created = MetaData.objects.update_or_create( - object_id=content_object.id, content_type=content_type, - data_type=data_type, defaults=defaults) + result, _created = MetaData.objects.update_or_create( + object_id=content_object.id, + content_type=content_type, + data_type=data_type, + defaults=defaults, + ) if data_file: - if result.data_value is None or result.data_value == '': + if result.data_value is None or result.data_value == "": result.data_value = data_file.name result.data_file = data_file result.data_file_type = data_file.content_type @@ -106,20 +124,24 @@ def unique_type_for_form(content_object, def type_for_form(content_object, data_type): + """ + Returns the MetaData queryset for ``content_object`` of the given ``data_type``. + """ content_type = ContentType.objects.get_for_model(content_object) - return MetaData.objects.filter(object_id=content_object.id, - content_type=content_type, - data_type=data_type) + return MetaData.objects.filter( + object_id=content_object.id, content_type=content_type, data_type=data_type + ) def create_media(media): """Download media link""" if is_valid_url(media.data_value): - filename = media.data_value.split('/')[-1] + filename = media.data_value.split("/")[-1] data_file = NamedTemporaryFile() content_type = mimetypes.guess_type(filename) - with closing(requests.get(media.data_value, stream=True)) as r: - for chunk in r.iter_content(chunk_size=CHUNK_SIZE): + with closing(requests.get(media.data_value, stream=True)) as resp: + # pylint: disable=no-member + for chunk in resp.iter_content(chunk_size=CHUNK_SIZE): if chunk: data_file.write(chunk) data_file.seek(os.SEEK_SET, os.SEEK_END) @@ -127,8 +149,8 @@ def create_media(media): data_file.seek(os.SEEK_SET) media.data_value = filename media.data_file = InMemoryUploadedFile( - data_file, 'data_file', filename, content_type, - size, charset=None) + data_file, "data_file", filename, content_type, size, charset=None + ) return media @@ -147,7 +169,7 @@ def media_resources(media_list, download=False): """ data = [] for media in media_list: - if media.data_file.name == '' and download: + if media.data_file.name == "" and download: media = create_media(media) if media: @@ -158,7 +180,10 @@ def media_resources(media_list, download=False): return data +# pylint: disable=too-many-public-methods class MetaData(models.Model): + """MetaData class model.""" + data_type = models.CharField(max_length=255) data_value = models.CharField(max_length=255) data_file = models.FileField(upload_to=upload_to, blank=True, null=True) @@ -167,52 +192,61 @@ class MetaData(models.Model): date_created = models.DateTimeField(null=True, auto_now_add=True) date_modified = models.DateTimeField(null=True, auto_now=True) deleted_at = models.DateTimeField(null=True, default=None) - content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, - default=get_default_content_type) + content_type = models.ForeignKey( + ContentType, on_delete=models.CASCADE, default=get_default_content_type + ) object_id = models.PositiveIntegerField(null=True, blank=True) - content_object = GenericForeignKey('content_type', 'object_id') + content_object = GenericForeignKey("content_type", "object_id") objects = models.Manager() class Meta: - app_label = 'main' - unique_together = ('object_id', 'data_type', 'data_value', - 'content_type') + app_label = "main" + unique_together = ("object_id", "data_type", "data_value", "content_type") + # pylint: disable=arguments-differ def save(self, *args, **kwargs): - self._set_hash() - super(MetaData, self).save(*args, **kwargs) + self.set_hash() + super().save(*args, **kwargs) @property def hash(self): - if self.file_hash is not None and self.file_hash != '': + """ + Returns the md5 hash of the metadata file. + """ + if self.file_hash is not None and self.file_hash != "": return self.file_hash - else: - return self._set_hash() - def _set_hash(self): + return self.set_hash() + + def set_hash(self): + """ + Returns the md5 hash of the metadata file. + """ if not self.data_file: return None file_exists = self.data_file.storage.exists(self.data_file.name) - if (file_exists and self.data_file.name != '') \ - or (not file_exists and self.data_file): + if (file_exists and self.data_file.name != "") or ( + not file_exists and self.data_file + ): try: self.data_file.seek(os.SEEK_SET) except IOError: - return '' + return "" else: - self.file_hash = 'md5:%s' % md5( - self.data_file.read()).hexdigest() + file_hash = hashlib.new( + "md5", self.data_file.read(), usedforsecurity=False + ).hexdigest() + self.file_hash = f"md5:{file_hash}" return self.file_hash - return '' + return "" def soft_delete(self): """ - Return the soft deletion timestamp Mark the MetaData as soft deleted, by updating the deleted_at field. """ @@ -222,15 +256,17 @@ def soft_delete(self): @staticmethod def public_link(content_object, data_value=None): - data_type = 'public_link' + """Returns the public link metadata.""" + data_type = "public_link" if data_value is False: - data_value = 'False' + data_value = "False" metadata = unique_type_for_form(content_object, data_type, data_value) # make text field a boolean - return metadata and metadata.data_value == 'True' + return metadata and metadata.data_value == "True" @staticmethod def set_google_sheet_details(content_object, data_value=None): + """Returns Google Sheet details metadata object.""" data_type = GOOGLE_SHEET_DATA_TYPE return unique_type_for_form(content_object, data_type, data_value) @@ -243,7 +279,7 @@ def get_google_sheet_details(obj): :param content_object_pk: xform primary key :return dictionary containing google sheet details """ - if isinstance(obj, basestring): + if isinstance(obj, str): metadata_data_value = obj else: metadata = MetaData.objects.filter( @@ -252,170 +288,220 @@ def get_google_sheet_details(obj): metadata_data_value = metadata and metadata.data_value if metadata_data_value: - data_list = metadata_data_value.split('|') + data_list = metadata_data_value.split("|") if data_list: # the data_list format is something like ['A a', 'B b c'] and # the list comprehension and dict cast results to # {'A': 'a', 'B': 'b c'} - return dict( - [tuple(a.strip().split(' ', 1)) for a in data_list]) + return dict([tuple(a.strip().split(" ", 1)) for a in data_list]) + + return metadata_data_value @staticmethod def published_by_formbuilder(content_object, data_value=None): - data_type = 'published_by_formbuilder' + """ + Returns the metadata object where data_type is 'published_by_formbuilder' + """ + data_type = "published_by_formbuilder" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def enketo_url(content_object, data_value=None): - data_type = 'enketo_url' + """ + Returns the metadata object where data_type is 'enket_url' + """ + data_type = "enketo_url" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def enketo_preview_url(content_object, data_value=None): - data_type = 'enketo_preview_url' + """ + Returns the metadata object where data_type is 'enketo_preview_url' + """ + data_type = "enketo_preview_url" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def enketo_single_submit_url(content_object, data_value=None): - data_type = 'enketo_single_submit_url' + """ + Returns the metadata object where data_type is 'enketo_single_submit_url' + """ + data_type = "enketo_single_submit_url" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def form_license(content_object, data_value=None): - data_type = 'form_license' + """ + Returns the metadata object where data_type is 'form_license' + """ + data_type = "form_license" obj = unique_type_for_form(content_object, data_type, data_value) - return (obj and obj.data_value) or None + return obj.data_value if obj else None @staticmethod def data_license(content_object, data_value=None): - data_type = 'data_license' + """ + Returns the metadata object where data_type is 'data_license' + """ + data_type = "data_license" obj = unique_type_for_form(content_object, data_type, data_value) - return (obj and obj.data_value) or None + return obj.data_value if obj else None @staticmethod def source(content_object, data_value=None, data_file=None): - data_type = 'source' - return unique_type_for_form( - content_object, data_type, data_value, data_file) + """ + Returns the metadata object where data_type is 'source' + """ + data_type = "source" + return unique_type_for_form(content_object, data_type, data_value, data_file) @staticmethod def supporting_docs(content_object, data_file=None): - data_type = 'supporting_doc' + """ + Returns the metadata object where data_type is 'supporting_doc' + """ + data_type = "supporting_doc" if data_file: content_type = ContentType.objects.get_for_model(content_object) - doc, created = MetaData.objects.update_or_create( + _doc, _created = MetaData.objects.update_or_create( data_type=data_type, content_type=content_type, object_id=content_object.id, data_value=data_file.name, defaults={ - 'data_file': data_file, - 'data_file_type': data_file.content_type - }) + "data_file": data_file, + "data_file_type": data_file.content_type, + }, + ) return type_for_form(content_object, data_type) @staticmethod def media_upload(content_object, data_file=None, download=False): - data_type = 'media' + """ + Returns the metadata object where data_type is 'media' + """ + data_type = "media" if data_file: allowed_types = settings.SUPPORTED_MEDIA_UPLOAD_TYPES - data_content_type = data_file.content_type \ - if data_file.content_type in allowed_types else \ - mimetypes.guess_type(data_file.name)[0] + data_content_type = ( + data_file.content_type + if data_file.content_type in allowed_types + else mimetypes.guess_type(data_file.name)[0] + ) if data_content_type in allowed_types: - content_type = ContentType.objects.get_for_model( - content_object) + content_type = ContentType.objects.get_for_model(content_object) - media, created = MetaData.objects.update_or_create( + _media, _created = MetaData.objects.update_or_create( data_type=data_type, content_type=content_type, object_id=content_object.id, data_value=data_file.name, defaults={ - 'data_file': data_file, - 'data_file_type': data_content_type - }) - return media_resources( - type_for_form(content_object, data_type), download) + "data_file": data_file, + "data_file_type": data_content_type, + }, + ) + return media_resources(type_for_form(content_object, data_type), download) @staticmethod def media_add_uri(content_object, uri): """Add a uri as a media resource""" - data_type = 'media' + data_type = "media" if is_valid_url(uri): - media, created = MetaData.objects.update_or_create( + _media, _created = MetaData.objects.update_or_create( data_type=data_type, data_value=uri, defaults={ - 'content_object': content_object, - }) + "content_object": content_object, + }, + ) @staticmethod def mapbox_layer_upload(content_object, data=None): - data_type = 'mapbox_layer' - if data and not MetaData.objects.filter(object_id=content_object.id, - data_type='mapbox_layer'): - s = '' + """ + Returns the metadata object where data_type is 'mapbox_layer' + """ + data_type = "mapbox_layer" + if data and not MetaData.objects.filter( + object_id=content_object.id, data_type="mapbox_layer" + ): + data_value = "" for key in data: - s = s + data[key] + '||' + data_value = data_value + data[key] + "||" content_type = ContentType.objects.get_for_model(content_object) - mapbox_layer = MetaData(data_type=data_type, - content_type=content_type, - object_id=content_object.id, - data_value=s) + mapbox_layer = MetaData( + data_type=data_type, + content_type=content_type, + object_id=content_object.id, + data_value=data_value, + ) mapbox_layer.save() if type_for_form(content_object, data_type): - values = type_for_form( - content_object, data_type)[0].data_value.split('||') + values = type_for_form(content_object, data_type)[0].data_value.split("||") data_values = {} - data_values['map_name'] = values[0] - data_values['link'] = values[1] - data_values['attribution'] = values[2] - data_values['id'] = type_for_form(content_object, data_type)[0].id + data_values["map_name"] = values[0] + data_values["link"] = values[1] + data_values["attribution"] = values[2] + data_values["id"] = type_for_form(content_object, data_type)[0].id return data_values - else: - return None + + return None @staticmethod def external_export(content_object, data_value=None): - data_type = 'external_export' + """ + Returns the metadata object where data_type is 'external_export' + """ + data_type = "external_export" if data_value: content_type = ContentType.objects.get_for_model(content_object) - result = MetaData(data_type=data_type, - content_type=content_type, - object_id=content_object.id, - data_value=data_value) + result = MetaData( + data_type=data_type, + content_type=content_type, + object_id=content_object.id, + data_value=data_value, + ) result.save() return result return MetaData.objects.filter( - object_id=content_object.id, data_type=data_type).order_by('-id') + object_id=content_object.id, data_type=data_type + ).order_by("-id") @property def external_export_url(self): - parts = self.data_value.split('|') + """ + Returns the external export URL + """ + parts = self.data_value.split("|") return parts[1] if len(parts) > 1 else None @property def external_export_name(self): - parts = self.data_value.split('|') + """ + Returns the external export name + """ + parts = self.data_value.split("|") return parts[0] if len(parts) > 1 else None @property def external_export_template(self): - parts = self.data_value.split('|') + """ + Returns the exxernal export, "XLS report", template + """ + parts = self.data_value.split("|") - return parts[1].replace('xls', 'templates') if len(parts) > 1 else None + return parts[1].replace("xls", "templates") if len(parts) > 1 else None @staticmethod def textit(content_object, data_value=None): @@ -427,49 +513,77 @@ def textit(content_object, data_value=None): @staticmethod def textit_flow_details(content_object, data_value: str = ""): + """ + Returns the metadata object where data_type is 'textit_details' + """ data_type = TEXTIT_DETAILS return unique_type_for_form(content_object, data_type, data_value) @property def is_linked_dataset(self): - return ( - isinstance(self.data_value, basestring) and - (self.data_value.startswith('xform') or - self.data_value.startswith('dataview')) + """ + Returns True if the metadata object is a linked dataset. + """ + return isinstance(self.data_value, str) and ( + self.data_value.startswith("xform") + or self.data_value.startswith("dataview") ) @staticmethod def xform_meta_permission(content_object, data_value=None): + """ + Returns the metadata object where data_type is 'xform_meta_perms' + """ data_type = XFORM_META_PERMS - return unique_type_for_form( - content_object, data_type, data_value) + return unique_type_for_form(content_object, data_type, data_value) @staticmethod def submission_review(content_object, data_value=None): - data_type = 'submission_review' + """ + Returns the metadata object where data_type is 'submission_review' + """ + data_type = "submission_review" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def instance_csv_imported_by(content_object, data_value=None): - data_type = 'imported_via_csv_by' + """ + Returns the metadata object where data_type is 'imported_via_csv_by' + """ + data_type = "imported_via_csv_by" return unique_type_for_form(content_object, data_type, data_value) +# pylint: disable=unused-argument,invalid-name def clear_cached_metadata_instance_object( - sender, instance=None, created=False, **kwargs): - safe_delete('{}{}'.format( - XFORM_METADATA_CACHE, instance.object_id)) + sender, instance=None, created=False, **kwargs +): + """ + Clear the cache for the metadata object. + """ + safe_delete(f"{XFORM_METADATA_CACHE}{instance.object_id}") +# pylint: disable=unused-argument def update_attached_object(sender, instance=None, created=False, **kwargs): + """ + Save the content_object attached to a MetaData instance. + """ if instance: instance.content_object.save() -post_save.connect(clear_cached_metadata_instance_object, sender=MetaData, - dispatch_uid='clear_cached_metadata_instance_object') -post_save.connect(update_attached_object, sender=MetaData, - dispatch_uid='update_attached_xform') -post_delete.connect(clear_cached_metadata_instance_object, sender=MetaData, - dispatch_uid='clear_cached_metadata_instance_delete') +post_save.connect( + clear_cached_metadata_instance_object, + sender=MetaData, + dispatch_uid="clear_cached_metadata_instance_object", +) +post_save.connect( + update_attached_object, sender=MetaData, dispatch_uid="update_attached_xform" +) +post_delete.connect( + clear_cached_metadata_instance_object, + sender=MetaData, + dispatch_uid="clear_cached_metadata_instance_delete", +) diff --git a/onadata/apps/main/models/user_profile.py b/onadata/apps/main/models/user_profile.py index 691a2d1a8c..c3cc98528e 100644 --- a/onadata/apps/main/models/user_profile.py +++ b/onadata/apps/main/models/user_profile.py @@ -2,35 +2,38 @@ """ UserProfile model class """ -import requests from django.conf import settings -from django.contrib.auth.models import User -from django.contrib.postgres.fields import JSONField +from django.contrib.auth import get_user_model from django.db import models from django.db.models.signals import post_save, pre_save -from django.utils.translation import ugettext_lazy -from django.utils.encoding import python_2_unicode_compatible -from guardian.shortcuts import get_perms_for_model, assign_perm -from guardian.models import UserObjectPermissionBase -from guardian.models import GroupObjectPermissionBase +from django.utils.translation import gettext_lazy + +import requests +from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase +from guardian.shortcuts import assign_perm, get_perms_for_model from rest_framework.authtoken.models import Token + +from onadata.apps.main.signals import ( + send_activation_email, + send_inactive_user_email, + set_api_permissions, +) from onadata.libs.utils.country_field import COUNTRIES from onadata.libs.utils.gravatar import get_gravatar_img_link, gravatar_exists -from onadata.apps.main.signals import ( - set_api_permissions, send_inactive_user_email, send_activation_email) -REQUIRE_AUTHENTICATION = 'REQUIRE_ODK_AUTHENTICATION' +REQUIRE_AUTHENTICATION = "REQUIRE_ODK_AUTHENTICATION" + +# pylint: disable=invalid-name +User = get_user_model() -@python_2_unicode_compatible class UserProfile(models.Model): """ Userprofile model """ # This field is required. - user = models.OneToOneField( - User, related_name='profile', on_delete=models.CASCADE) + user = models.OneToOneField(User, related_name="profile", on_delete=models.CASCADE) # Other fields here name = models.CharField(max_length=255, blank=True) @@ -41,58 +44,76 @@ class UserProfile(models.Model): twitter = models.CharField(max_length=255, blank=True) description = models.CharField(max_length=255, blank=True) require_auth = models.BooleanField( - default=False, - verbose_name=ugettext_lazy("Require Phone Authentication")) + default=False, verbose_name=gettext_lazy("Require Phone Authentication") + ) address = models.CharField(max_length=255, blank=True) phonenumber = models.CharField(max_length=30, blank=True) created_by = models.ForeignKey( - User, null=True, blank=True, on_delete=models.SET_NULL) + User, null=True, blank=True, on_delete=models.SET_NULL + ) num_of_submissions = models.IntegerField(default=0) - metadata = JSONField(default=dict, blank=True) + metadata = models.JSONField(default=dict, blank=True) date_modified = models.DateTimeField(auto_now=True) def __str__(self): - return u'%s[%s]' % (self.name, self.user.username) + return f"{self.name}[{self.user.username}]" @property def gravatar(self): + """ + Returns Gravatar URL. + """ return get_gravatar_img_link(self.user) @property def gravatar_exists(self): + """ + Check if Gravatar URL exists. + """ return gravatar_exists(self.user) @property def twitter_clean(self): + """ + Remove the '@' from twitter name. + """ if self.twitter.startswith("@"): return self.twitter[1:] return self.twitter - def save(self, force_insert=False, force_update=False, using=None, - update_fields=None): + def save( + self, force_insert=False, force_update=False, using=None, update_fields=None + ): # Override default save method to set settings configured require_auth # value if self.pk is None and hasattr(settings, REQUIRE_AUTHENTICATION): self.require_auth = getattr(settings, REQUIRE_AUTHENTICATION) - super(UserProfile, self).save(force_insert, force_update, using, - update_fields) + super().save(force_insert, force_update, using, update_fields) class Meta: - app_label = 'main' + app_label = "main" permissions = ( - ('can_add_project', "Can add a project to an organization"), - ('can_add_xform', "Can add/upload an xform to user profile"), - ('view_profile', "Can view user profile"), + ("can_add_project", "Can add a project to an organization"), + ("can_add_xform", "Can add/upload an xform to user profile"), + ("view_profile", "Can view user profile"), ) +# pylint: disable=unused-argument def create_auth_token(sender, instance=None, created=False, **kwargs): + """ + Creates an authentication Token. + """ if created: Token.objects.create(user=instance) +# pylint: disable=unused-argument def set_object_permissions(sender, instance=None, created=False, **kwargs): + """ + Assign's permission to the user that created the profile. + """ if created: for perm in get_perms_for_model(UserProfile): assign_perm(perm.codename, instance.user, instance) @@ -101,47 +122,52 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): assign_perm(perm.codename, instance.created_by, instance) -def set_kpi_formbuilder_permissions( - sender, instance=None, created=False, **kwargs): +# pylint: disable=unused-argument +def set_kpi_formbuilder_permissions(sender, instance=None, created=False, **kwargs): + """ + Assign KPI permissions to allow the user to create forms using KPI formbuilder. + """ if created: - kpi_formbuilder_url = hasattr(settings, 'KPI_FORMBUILDER_URL') and\ - settings.KPI_FORMBUILDER_URL + kpi_formbuilder_url = ( + hasattr(settings, "KPI_FORMBUILDER_URL") and settings.KPI_FORMBUILDER_URL + ) if kpi_formbuilder_url: requests.post( - "%s/%s" % ( - kpi_formbuilder_url, - 'grant-default-model-level-perms' - ), - headers={ - 'Authorization': 'Token %s' % instance.user.auth_token - } + f"{kpi_formbuilder_url}/grant-default-model-level-perms", + headers={"Authorization": "Token {instance.user.auth_token}"}, ) -post_save.connect(create_auth_token, sender=User, dispatch_uid='auth_token') +post_save.connect(create_auth_token, sender=User, dispatch_uid="auth_token") post_save.connect( - send_inactive_user_email, sender=User, - dispatch_uid='send_inactive_user_email') + send_inactive_user_email, sender=User, dispatch_uid="send_inactive_user_email" +) pre_save.connect( - send_activation_email, sender=User, - dispatch_uid='send_activation_email' + send_activation_email, sender=User, dispatch_uid="send_activation_email" ) -post_save.connect(set_api_permissions, sender=User, - dispatch_uid='set_api_permissions') +post_save.connect(set_api_permissions, sender=User, dispatch_uid="set_api_permissions") -post_save.connect(set_object_permissions, sender=UserProfile, - dispatch_uid='set_object_permissions') +post_save.connect( + set_object_permissions, sender=UserProfile, dispatch_uid="set_object_permissions" +) -post_save.connect(set_kpi_formbuilder_permissions, sender=UserProfile, - dispatch_uid='set_kpi_formbuilder_permission') +post_save.connect( + set_kpi_formbuilder_permissions, + sender=UserProfile, + dispatch_uid="set_kpi_formbuilder_permission", +) +# pylint: disable=too-few-public-methods class UserProfileUserObjectPermission(UserObjectPermissionBase): """Guardian model to create direct foreign keys.""" + content_object = models.ForeignKey(UserProfile, on_delete=models.CASCADE) +# pylint: disable=too-few-public-methods class UserProfileGroupObjectPermission(GroupObjectPermissionBase): """Guardian model to create direct foreign keys.""" + content_object = models.ForeignKey(UserProfile, on_delete=models.CASCADE) diff --git a/onadata/apps/main/registration_urls.py b/onadata/apps/main/registration_urls.py index 7038f19306..ff1eee5ece 100644 --- a/onadata/apps/main/registration_urls.py +++ b/onadata/apps/main/registration_urls.py @@ -8,7 +8,7 @@ """ -from django.conf.urls import url, include +from django.urls import include, path, re_path from django.views.generic import TemplateView from registration.backends.default.views import ActivationView @@ -16,23 +16,29 @@ from onadata.apps.main.forms import RegistrationFormUserProfile urlpatterns = [ - url(r'^activate/complete/$', - TemplateView.as_view( - template_name='registration/activation_complete.html'), - name='registration_activation_complete'), + path( + "activate/complete/", + TemplateView.as_view(template_name="registration/activation_complete.html"), + name="registration_activation_complete", + ), # Activation keys get matched by \w+ instead of the more specific # [a-fA-F0-9]{40} because a bad activation key should still get to the view # that way it can return a sensible "invalid key" message instead of a # confusing 404. - url(r'^activate/(?P\w+)/$', + re_path( + r"^activate/(?P\w+)/$", ActivationView.as_view(), - name='registration_activate'), - url(r'^register/$', + name="registration_activate", + ), + path( + "register/", FHRegistrationView.as_view(form_class=RegistrationFormUserProfile), - name='registration_register'), - url(r'^register/complete/$', - TemplateView.as_view( - template_name='registration/registration_complete.html'), - name='registration_complete'), - url(r'', include('registration.auth_urls')), + name="registration_register", + ), + path( + "register/complete/", + TemplateView.as_view(template_name="registration/registration_complete.html"), + name="registration_complete", + ), + re_path(r"", include("registration.auth_urls")), ] diff --git a/onadata/apps/main/tests/test_base.py b/onadata/apps/main/tests/test_base.py index abc963ef1b..deee090673 100644 --- a/onadata/apps/main/tests/test_base.py +++ b/onadata/apps/main/tests/test_base.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +TestBase - a TestCase base class. +""" from __future__ import unicode_literals import base64 @@ -5,23 +9,21 @@ import os import re import socket -from builtins import open -from future.moves.urllib.error import URLError -from future.moves.urllib.request import urlopen from io import StringIO from tempfile import NamedTemporaryFile from django.conf import settings -from django.contrib.auth import authenticate -from django.contrib.auth.models import User +from django.contrib.auth import authenticate, get_user_model from django.core.files.uploadedfile import InMemoryUploadedFile from django.test import RequestFactory, TransactionTestCase from django.test.client import Client from django.utils import timezone +from six.moves.urllib.error import URLError +from six.moves.urllib.request import urlopen + from django_digest.test import Client as DigestClient from django_digest.test import DigestAuth -from onadata.libs.test_utils.pyxform_test_case import PyxformMarkdown from rest_framework.test import APIRequestFactory from onadata.apps.api.viewsets.xform_viewset import XFormViewSet @@ -29,30 +31,45 @@ from onadata.apps.logger.views import submission from onadata.apps.main.models import UserProfile from onadata.apps.viewer.models import DataDictionary -from onadata.libs.utils.common_tools import (filename_from_disposition, - get_response_content) +from onadata.libs.test_utils.pyxform_test_case import PyxformMarkdown +from onadata.libs.utils.common_tools import ( + filename_from_disposition, + get_response_content, +) from onadata.libs.utils.user_auth import get_user_default_project +# pylint: disable=invalid-name +User = get_user_model() + +# pylint: disable=too-many-instance-attributes class TestBase(PyxformMarkdown, TransactionTestCase): + """ + A TransactionTestCase base class for test modules. + """ + maxDiff = None - surveys = ['transport_2011-07-25_19-05-49', - 'transport_2011-07-25_19-05-36', - 'transport_2011-07-25_19-06-01', - 'transport_2011-07-25_19-06-14'] + surveys = [ + "transport_2011-07-25_19-05-49", + "transport_2011-07-25_19-05-36", + "transport_2011-07-25_19-06-01", + "transport_2011-07-25_19-06-14", + ] this_directory = os.path.abspath(os.path.dirname(__file__)) def setUp(self): self.maxDiff = None self._create_user_and_login() - self.base_url = 'http://testserver' + self.base_url = "http://testserver" self.factory = RequestFactory() + # pylint: disable=no-self-use def _fixture_path(self, *args): - return os.path.join(os.path.dirname(__file__), 'fixtures', *args) + return os.path.join(os.path.dirname(__file__), "fixtures", *args) + # pylint: disable=no-self-use def _create_user(self, username, password, create_profile=False): - user, created = User.objects.get_or_create(username=username) + user, _created = User.objects.get_or_create(username=username) user.set_password(password) user.save() @@ -64,6 +81,7 @@ def _create_user(self, username, password, create_profile=False): return user + # pylint: disable=no-self-use def _login(self, username, password): client = Client() assert client.login(username=username, password=password) @@ -74,8 +92,7 @@ def _logout(self, client=None): client = self.client client.logout() - def _create_user_and_login(self, username="bob", password="bob", - factory=None): + def _create_user_and_login(self, username="bob", password="bob", factory=None): self.login_username = username self.login_password = password self.user = self._create_user(username, password, create_profile=True) @@ -85,34 +102,36 @@ def _create_user_and_login(self, username="bob", password="bob", self.anon = Client() def _publish_xls_file(self, path): - if not path.startswith('/%s/' % self.user.username): + if not path.startswith(f"/{self.user.username}/"): path = os.path.join(self.this_directory, path) - with open(path, 'rb') as f: + with open(path, "rb") as f: xls_file = InMemoryUploadedFile( f, - 'xls_file', + "xls_file", os.path.abspath(os.path.basename(path)), - 'application/vnd.ms-excel', + "application/vnd.ms-excel", os.path.getsize(path), - None) - if not hasattr(self, 'project'): + None, + ) + if not hasattr(self, "project"): + # pylint: disable=attribute-defined-outside-init self.project = get_user_default_project(self.user) DataDictionary.objects.create( - created_by=self.user, - user=self.user, - xls=xls_file, - project=self.project) + created_by=self.user, user=self.user, xls=xls_file, project=self.project + ) def _publish_xlsx_file(self): - path = os.path.join(self.this_directory, 'fixtures', 'exp.xlsx') + path = os.path.join(self.this_directory, "fixtures", "exp.xlsx") pre_count = XForm.objects.count() TestBase._publish_xls_file(self, path) # make sure publishing the survey worked self.assertEqual(XForm.objects.count(), pre_count + 1) def _publish_xlsx_file_with_external_choices(self): - path = os.path.join(self.this_directory, 'fixtures', 'external_choice_form_v1.xlsx') + path = os.path.join( + self.this_directory, "fixtures", "external_choice_form_v1.xlsx" + ) pre_count = XForm.objects.count() TestBase._publish_xls_file(self, path) # make sure publishing the survey worked @@ -122,41 +141,71 @@ def _publish_xls_file_and_set_xform(self, path): count = XForm.objects.count() self._publish_xls_file(path) self.assertEqual(XForm.objects.count(), count + 1) - self.xform = XForm.objects.order_by('pk').reverse()[0] + # pylint: disable=attribute-defined-outside-init + self.xform = XForm.objects.order_by("pk").reverse()[0] - def _share_form_data(self, id_string='transportation_2011_07_25'): + def _share_form_data(self, id_string="transportation_2011_07_25"): xform = XForm.objects.get(id_string=id_string) xform.shared_data = True xform.save() def _publish_transportation_form(self): xls_path = os.path.join( - self.this_directory, "fixtures", - "transportation", "transportation.xlsx") + self.this_directory, "fixtures", "transportation", "transportation.xlsx" + ) count = XForm.objects.count() TestBase._publish_xls_file(self, xls_path) self.assertEqual(XForm.objects.count(), count + 1) - self.xform = XForm.objects.order_by('pk').reverse()[0] + # pylint: disable=attribute-defined-outside-init + self.xform = XForm.objects.order_by("pk").reverse()[0] def _submit_transport_instance(self, survey_at=0): s = self.surveys[survey_at] - self._make_submission(os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml')) + self._make_submission( + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + ) def _submit_transport_instance_w_uuid(self, name): - self._make_submission(os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances_w_uuid', name, name + '.xml')) + self._make_submission( + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances_w_uuid", + name, + name + ".xml", + ) + ) def _submit_transport_instance_w_attachment(self, survey_at=0): s = self.surveys[survey_at] media_file = "1335783522563.jpg" - self._make_submission_w_attachment(os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml'), - os.path.join(self.this_directory, 'fixtures', - 'transportation', 'instances', s, media_file)) + self._make_submission_w_attachment( + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ), + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ), + ) + # pylint: disable=attribute-defined-outside-init self.attachment = Attachment.objects.all().reverse()[0] self.attachment_media_file = self.attachment.media_file @@ -165,49 +214,56 @@ def _publish_transportation_form_and_submit_instance(self): self._submit_transport_instance() def _make_submissions_gps(self): - surveys = ['gps_1980-01-23_20-52-08', - 'gps_1980-01-23_21-21-33', ] + surveys = [ + "gps_1980-01-23_20-52-08", + "gps_1980-01-23_21-21-33", + ] for survey in surveys: - path = self._fixture_path('gps', 'instances', survey + '.xml') + path = self._fixture_path("gps", "instances", survey + ".xml") self._make_submission(path) - def _make_submission(self, path, username=None, add_uuid=False, - forced_submission_time=None, auth=None, client=None): + # pylint: disable=too-many-arguments,too-many-locals,unused-argument + def _make_submission( + self, + path, + username=None, + add_uuid=False, + forced_submission_time=None, + auth=None, + client=None, + ): # store temporary file with dynamic uuid self.factory = APIRequestFactory() if auth is None: - auth = DigestAuth('bob', 'bob') + auth = DigestAuth("bob", "bob") tmp_file = None if add_uuid: - tmp_file = NamedTemporaryFile(delete=False, mode='w') - split_xml = None + with NamedTemporaryFile(delete=False, mode="w") as tmp_file: + split_xml = None - with open(path, encoding='utf-8') as _file: - split_xml = re.split(r'()', _file.read()) + with open(path, encoding="utf-8") as _file: + split_xml = re.split(r"()", _file.read()) - split_xml[1:1] = [ - '%s' % self.xform.uuid - ] - tmp_file.write(''.join(split_xml)) - path = tmp_file.name - tmp_file.close() + split_xml[1:1] = [f"{self.xform.uuid}"] + tmp_file.write("".join(split_xml)) + path = tmp_file.name - with open(path, encoding='utf-8') as f: - post_data = {'xml_submission_file': f} + with open(path, encoding="utf-8") as f: + post_data = {"xml_submission_file": f} if username is None: username = self.user.username - url_prefix = '%s/' % username if username else '' - url = '/%ssubmission' % url_prefix + url_prefix = f"{username if username else ''}/" + url = f"/{url_prefix}submission" request = self.factory.post(url, post_data) - request.user = authenticate(username=auth.username, - password=auth.password) + request.user = authenticate(username=auth.username, password=auth.password) + # pylint: disable=attribute-defined-outside-init self.response = submission(request, username=username) if auth and self.response.status_code == 401: @@ -215,7 +271,7 @@ def _make_submission(self, path, username=None, add_uuid=False, self.response = submission(request, username=username) if forced_submission_time: - instance = Instance.objects.order_by('-pk').all()[0] + instance = Instance.objects.order_by("-pk").all()[0] instance.date_created = forced_submission_time instance.json = instance.get_full_dict() instance.save() @@ -226,33 +282,30 @@ def _make_submission(self, path, username=None, add_uuid=False, os.unlink(tmp_file.name) def _make_submission_w_attachment(self, path, attachment_path): - with open(path, encoding='utf-8') as f: - data = {'xml_submission_file': f} + with open(path, encoding="utf-8") as f: + data = {"xml_submission_file": f} if attachment_path is not None: if isinstance(attachment_path, list): - for c in range(len(attachment_path)): - data['media_file_{}'.format(c)] = open( - attachment_path[c], 'rb') + for index, item_path in enumerate(attachment_path): + # pylint: disable=consider-using-with + data[f"media_file_{index}"] = open(item_path, "rb") else: - data['media_file'] = open( - attachment_path, 'rb') + # pylint: disable=consider-using-with + data["media_file"] = open(attachment_path, "rb") - url = '/%s/submission' % self.user.username - auth = DigestAuth('bob', 'bob') + url = f"/{self.user.username}/submission" + auth = DigestAuth("bob", "bob") self.factory = APIRequestFactory() request = self.factory.post(url, data) - request.user = authenticate(username='bob', - password='bob') - self.response = submission(request, - username=self.user.username) + request.user = authenticate(username="bob", password="bob") + # pylint: disable=attribute-defined-outside-init + self.response = submission(request, username=self.user.username) if auth and self.response.status_code == 401: request.META.update(auth(request.META, self.response)) - self.response = submission(request, - username=self.user.username) + self.response = submission(request, username=self.user.username) - def _make_submissions(self, username=None, add_uuid=False, - should_store=True): + def _make_submissions(self, username=None, add_uuid=False, should_store=True): """Make test fixture submissions to current xform. :param username: submit under this username, default None. @@ -260,16 +313,23 @@ def _make_submissions(self, username=None, add_uuid=False, :param should_store: should submissions be save, default True. """ - paths = [os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'instances', s, s + '.xml') for s in self.surveys] + paths = [ + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + for s in self.surveys + ] pre_count = Instance.objects.count() for path in paths: self._make_submission(path, username, add_uuid) - post_count = pre_count + len(self.surveys) if should_store\ - else pre_count + post_count = pre_count + len(self.surveys) if should_store else pre_count self.assertEqual(Instance.objects.count(), post_count) self.assertEqual(self.xform.instances.count(), post_count) xform = XForm.objects.get(pk=self.xform.pk) @@ -278,32 +338,34 @@ def _make_submissions(self, username=None, add_uuid=False, def _check_url(self, url, timeout=1): try: - urlopen(url, timeout=timeout) - return True + with urlopen(url, timeout=timeout): + return True except (URLError, socket.timeout): pass return False - def _internet_on(self, url='http://74.125.113.99'): + def _internet_on(self, url="http://74.125.113.99"): # default value is some google IP return self._check_url(url) def _set_auth_headers(self, username, password): return { - 'HTTP_AUTHORIZATION': - 'Basic ' + base64.b64encode(( - '%s:%s' % (username, password)).encode( - 'utf-8')).decode('utf-8'), + "HTTP_AUTHORIZATION": "Basic " + + base64.b64encode(f"{username}:{password}".encode("utf-8")).decode( + "utf-8" + ), } def _get_authenticated_client( - self, url, username='bob', password='bob', extra={}): + self, url, username="bob", password="bob", extra=None + ): client = DigestClient() + extra = {} if extra is None else extra # request with no credentials req = client.get(url, {}, **extra) self.assertEqual(req.status_code, 401) # apply credentials - client.set_authorization(username, password, 'Digest') + client.set_authorization(username, password, "Digest") return client def _set_mock_time(self, mock_time): @@ -311,62 +373,79 @@ def _set_mock_time(self, mock_time): mock_time.return_value = current_time def _set_require_auth(self, auth=True): - profile, created = UserProfile.objects.get_or_create(user=self.user) + profile, _created = UserProfile.objects.get_or_create(user=self.user) profile.require_auth = auth profile.save() def _get_digest_client(self): self._set_require_auth(True) client = DigestClient() - client.set_authorization('bob', 'bob', 'Digest') + client.set_authorization("bob", "bob", "Digest") return client def _publish_submit_geojson(self): path = os.path.join( - settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", - "geolocation", "GeoLocationForm.xlsx") + settings.PROJECT_ROOT, + "apps", + "main", + "tests", + "fixtures", + "geolocation", + "GeoLocationForm.xlsx", + ) self._publish_xls_file_and_set_xform(path) - view = XFormViewSet.as_view({'post': 'csv_import'}) - csv_import = \ - open(os.path.join(settings.PROJECT_ROOT, 'apps', 'main', - 'tests', 'fixtures', 'geolocation', - 'GeoLocationForm_2015_01_15_01_28_45.csv'), - encoding='utf-8') - post_data = {'csv_file': csv_import} - request = self.factory.post('/', data=post_data, **self.extra) - response = view(request, pk=self.xform.id) - self.assertEqual(response.status_code, 200) + view = XFormViewSet.as_view({"post": "csv_import"}) + with open( + os.path.join( + settings.PROJECT_ROOT, + "apps", + "main", + "tests", + "fixtures", + "geolocation", + "GeoLocationForm_2015_01_15_01_28_45.csv", + ), + encoding="utf-8", + ) as csv_import: + post_data = {"csv_file": csv_import} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request, pk=self.xform.id) + self.assertEqual(response.status_code, 200) def _publish_markdown(self, md_xlsform, user, project=None, **kwargs): """ Publishes a markdown XLSForm. """ - kwargs['name'] = 'data' + kwargs["name"] = "data" survey = self.md_to_pyxform_survey(md_xlsform, kwargs=kwargs) - survey['sms_keyword'] = survey['id_string'] - if not project or not hasattr(self, 'project'): + survey["sms_keyword"] = survey["id_string"] + if not project or not hasattr(self, "project"): project = get_user_default_project(user) - xform = DataDictionary(created_by=user, user=user, - xml=survey.to_xml(), json=survey.to_json(), - project=project) + xform = DataDictionary( + created_by=user, + user=user, + xml=survey.to_xml(), + json=survey.to_json(), + project=project, + ) xform.save() return xform def _test_csv_response(self, response, csv_file_path): headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], 'application/csv') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/csv") + content_disposition = headers["Content-Disposition"] filename = filename_from_disposition(content_disposition) __, ext = os.path.splitext(filename) - self.assertEqual(ext, '.csv') + self.assertEqual(ext, ".csv") data = get_response_content(response) reader = csv.DictReader(StringIO(data)) - data = [_ for _ in reader] - with open(csv_file_path, encoding='utf-8') as test_file: + data = list(reader) + with open(csv_file_path, encoding="utf-8") as test_file: expected_csv_reader = csv.DictReader(test_file) for index, row in enumerate(expected_csv_reader): if None in row: @@ -375,8 +454,8 @@ def _test_csv_response(self, response, csv_file_path): def _test_csv_files(self, csv_file, csv_file_path): reader = csv.DictReader(csv_file) - data = [_ for _ in reader] - with open(csv_file_path, encoding='utf-8') as test_file: + data = list(reader) + with open(csv_file_path, encoding="utf-8") as test_file: expected_csv_reader = csv.DictReader(test_file) for index, row in enumerate(expected_csv_reader): if None in row: diff --git a/onadata/apps/main/tests/test_csv_export.py b/onadata/apps/main/tests/test_csv_export.py index f7b022fc7e..db1ca74bb1 100644 --- a/onadata/apps/main/tests/test_csv_export.py +++ b/onadata/apps/main/tests/test_csv_export.py @@ -73,7 +73,7 @@ def test_csv_nested_repeat_output(self): u'/data/bed_net[2]/member[2]/name', u'/data/meta/instanceID' ] - self.assertEquals(data_dictionary.xpaths(repeat_iterations=2), xpaths) + self.assertEqual(data_dictionary.xpaths(repeat_iterations=2), xpaths) # test csv export = generate_export( Export.CSV_EXPORT, diff --git a/onadata/apps/main/tests/test_form_auth.py b/onadata/apps/main/tests/test_form_auth.py index 45351c6195..3167918098 100644 --- a/onadata/apps/main/tests/test_form_auth.py +++ b/onadata/apps/main/tests/test_form_auth.py @@ -11,4 +11,4 @@ def setUp(self): def test_login_redirect_redirects(self): response = self.client.get(reverse(login_redirect)) - self.assertEquals(response.status_code, 302) + self.assertEqual(response.status_code, 302) diff --git a/onadata/apps/main/tests/test_form_enter_data.py b/onadata/apps/main/tests/test_form_enter_data.py index b27dbc51f4..0eb2b18186 100644 --- a/onadata/apps/main/tests/test_form_enter_data.py +++ b/onadata/apps/main/tests/test_form_enter_data.py @@ -7,10 +7,9 @@ from django.core.validators import URLValidator from django.test import RequestFactory from django.urls import reverse -from future.moves.urllib.parse import urlparse +from six.moves.urllib.parse import urlparse from httmock import HTTMock, urlmatch from nose import SkipTest -from past.builtins import basestring from onadata.apps.logger.views import enter_data from onadata.apps.main.models import MetaData @@ -39,9 +38,7 @@ def enketo_mock_http(url, request): def enketo_error_mock(url, request): response = requests.Response() response.status_code = 400 - response._content = ( - '{"message": "no account exists for this OpenRosa server"}' - ) + response._content = '{"message": "no account exists for this OpenRosa server"}' return response @@ -80,8 +77,8 @@ def test_enketo_remote_server(self): server_url = "https://testserver.com/bob" form_id = "test_%s" % re.sub(re.compile("\."), "_", str(time())) # noqa url = get_enketo_urls(server_url, form_id) - self.assertIsInstance(url['url'], basestring) - self.assertIsNone(URLValidator()(url['url'])) + self.assertIsInstance(url["url"], str) + self.assertIsNone(URLValidator()(url["url"])) def test_enketo_url_with_http_protocol_on_formlist(self): if not self._running_enketo(): @@ -90,9 +87,9 @@ def test_enketo_url_with_http_protocol_on_formlist(self): server_url = "http://testserver.com/bob" form_id = "test_%s" % re.sub(re.compile("\."), "_", str(time())) # noqa url = get_enketo_urls(server_url, form_id) - self.assertIn("http:", url['url']) - self.assertIsInstance(url['url'], basestring) - self.assertIsNone(URLValidator()(url['url'])) + self.assertIn("http:", url["url"]) + self.assertIsInstance(url["url"], str) + self.assertIsNone(URLValidator()(url["url"])) def _get_grcode_view_response(self): factory = RequestFactory() @@ -105,9 +102,7 @@ def _get_grcode_view_response(self): def test_qrcode_view(self): with HTTMock(enketo_mock): response = self._get_grcode_view_response() - self.assertContains( - response, "data:image/png;base64,", status_code=200 - ) + self.assertContains(response, "data:image/png;base64,", status_code=200) def test_qrcode_view_with_enketo_error(self): with HTTMock(enketo_error_mock): @@ -121,9 +116,7 @@ def test_enter_data_redir(self): factory = RequestFactory() request = factory.get("/") request.user = self.user - response = enter_data( - request, self.user.username, self.xform.id_string - ) + response = enter_data(request, self.user.username, self.xform.id_string) # make sure response redirect to an enketo site enketo_base_url = urlparse(settings.ENKETO_URL).netloc redirected_base_url = urlparse(response["Location"]).netloc @@ -157,9 +150,7 @@ def test_public_with_link_to_share_toggle_on(self): factory = RequestFactory() request = factory.get("/") request.user = AnonymousUser() - response = enter_data( - request, self.user.username, self.xform.id_string - ) + response = enter_data(request, self.user.username, self.xform.id_string) self.assertEqual(response.status_code, 302) def test_enter_data_non_existent_user(self): diff --git a/onadata/apps/main/tests/test_form_errors.py b/onadata/apps/main/tests/test_form_errors.py index d4992f72b1..77bbcf7003 100644 --- a/onadata/apps/main/tests/test_form_errors.py +++ b/onadata/apps/main/tests/test_form_errors.py @@ -28,7 +28,7 @@ def test_bad_id_string(self): xls_path = os.path.join(self.this_directory, "fixtures", "transportation", "transportation.bad_id.xlsx") self.assertRaises(XLSFormError, self._publish_xls_file, xls_path) - self.assertEquals(XForm.objects.count(), count) + self.assertEqual(XForm.objects.count(), count) @skip def test_dl_no_xls(self): @@ -99,7 +99,7 @@ def test_spaced_xlsform(self): " have modified the filename to not contain any spaces.") self.assertRaisesMessage( XLSFormError, msg, self._publish_xls_file, xls_path) - self.assertEquals(XForm.objects.count(), count) + self.assertEqual(XForm.objects.count(), count) def test_choice_duplicate_error(self): """ @@ -122,4 +122,4 @@ def test_choice_duplicate_error(self): "Learn more: https://xlsform.org/#choice-names.") self.assertRaisesMessage( PyXFormError, msg, self._publish_xls_file, xls_path) - self.assertEquals(XForm.objects.count(), count) + self.assertEqual(XForm.objects.count(), count) diff --git a/onadata/apps/main/tests/test_form_metadata.py b/onadata/apps/main/tests/test_form_metadata.py index 2eec07e1b1..a0e72c1e11 100644 --- a/onadata/apps/main/tests/test_form_metadata.py +++ b/onadata/apps/main/tests/test_form_metadata.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Test form MetaData objects. +""" import hashlib import os from builtins import open @@ -10,112 +14,147 @@ from onadata.apps.main.models import MetaData from onadata.apps.main.tests.test_base import TestBase -from onadata.apps.main.views import show, edit, download_metadata, \ - download_media_data, delete_metadata +from onadata.apps.main.views import ( + show, + edit, + download_metadata, + download_media_data, + delete_metadata, +) from onadata.libs.utils.cache_tools import XFORM_METADATA_CACHE +# pylint: disable=too-many-public-methods class TestFormMetadata(TestBase): + """ + Test form MetaData objects. + """ def setUp(self): TestBase.setUp(self) self._create_user_and_login() self._publish_transportation_form_and_submit_instance() - self.url = reverse(show, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - }) - self.edit_url = reverse(edit, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - }) - - def _add_metadata(self, data_type='doc'): - if data_type == 'media': - name = 'screenshot.png' + self.url = reverse( + show, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) + self.edit_url = reverse( + edit, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) + + def _add_metadata(self, data_type="doc"): + if data_type == "media": + name = "screenshot.png" else: - name = 'transportation.xlsx' - path = os.path.join(self.this_directory, "fixtures", - "transportation", name) - with open(path, 'rb') as doc_file: + name = "transportation.xlsx" + path = os.path.join(self.this_directory, "fixtures", "transportation", name) + with open(path, "rb") as doc_file: self.post_data = {} self.post_data[data_type] = doc_file self.client.post(self.edit_url, self.post_data) - if data_type == 'media': - self.doc = MetaData.objects.filter(data_type='media').reverse()[0] - self.doc_url = reverse(download_media_data, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - 'data_id': self.doc.id}) + if data_type == "media": + self.doc = MetaData.objects.filter(data_type="media").reverse()[0] + self.doc_url = reverse( + download_media_data, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + "data_id": self.doc.id, + }, + ) else: self.doc = MetaData.objects.all().reverse()[0] - self.doc_url = reverse(download_metadata, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - 'data_id': self.doc.id}) + self.doc_url = reverse( + download_metadata, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + "data_id": self.doc.id, + }, + ) return name def test_views_with_unavailable_id_string(self): path = os.path.join( - self.this_directory, "fixtures", "transportation", - 'transportation.xlsx' + self.this_directory, "fixtures", "transportation", "transportation.xlsx" ) - with open(path, 'rb') as doc_file: + with open(path, "rb") as doc_file: self.post_data = {} - self.post_data['doc'] = doc_file + self.post_data["doc"] = doc_file self.client.post(self.edit_url, self.post_data) self.doc = MetaData.objects.all().reverse()[0] - download_metadata_url = reverse(download_metadata, kwargs={ - 'username': self.user.username, - 'id_string': 'random_id_string', - 'data_id': self.doc.id}) + download_metadata_url = reverse( + download_metadata, + kwargs={ + "username": self.user.username, + "id_string": "random_id_string", + "data_id": self.doc.id, + }, + ) response = self.client.get(download_metadata_url) self.assertEqual(response.status_code, 404) - delete_metadata_url = reverse(delete_metadata, kwargs={ - 'username': self.user.username, - 'id_string': 'random_id_string', - 'data_id': self.doc.id}) + delete_metadata_url = reverse( + delete_metadata, + kwargs={ + "username": self.user.username, + "id_string": "random_id_string", + "data_id": self.doc.id, + }, + ) - response = self.client.get(delete_metadata_url + '?del=true') + response = self.client.get(delete_metadata_url + "?del=true") self.assertEqual(response.status_code, 404) def test_adds_supporting_doc_on_submit(self): - count = len(MetaData.objects.filter(object_id=self.xform.id, - data_type='supporting_doc')) + count = len( + MetaData.objects.filter(object_id=self.xform.id, data_type="supporting_doc") + ) self._add_metadata() - self.assertEquals(count + 1, len(MetaData.objects.filter( - object_id=self.xform.id, data_type='supporting_doc'))) + self.assertEqual( + count + 1, + len( + MetaData.objects.filter( + object_id=self.xform.id, data_type="supporting_doc" + ) + ), + ) def test_delete_cached_xform_metadata_object_on_save(self): count = MetaData.objects.count() - cache.set('{}{}'.format(XFORM_METADATA_CACHE, self.xform.id), True) + cache.set("{}{}".format(XFORM_METADATA_CACHE, self.xform.id), True) self._add_metadata() - self.assertIsNone( - cache.get('{}{}'.format(XFORM_METADATA_CACHE, self.xform.id))) - self.assertEquals(count + 1, MetaData.objects.count()) + self.assertIsNone(cache.get("{}{}".format(XFORM_METADATA_CACHE, self.xform.id))) + self.assertEqual(count + 1, MetaData.objects.count()) def test_adds_supporting_media_on_submit(self): - count = len(MetaData.objects.filter(object_id=self.xform.id, - data_type='media')) - self._add_metadata(data_type='media') - self.assertEquals(count + 1, len(MetaData.objects.filter( - object_id=self.xform.id, data_type='media'))) + count = len(MetaData.objects.filter(object_id=self.xform.id, data_type="media")) + self._add_metadata(data_type="media") + self.assertEqual( + count + 1, + len(MetaData.objects.filter(object_id=self.xform.id, data_type="media")), + ) def test_adds_mapbox_layer_on_submit(self): count = MetaData.objects.filter( - object_id=self.xform.id, data_type='mapbox_layer').count() + object_id=self.xform.id, data_type="mapbox_layer" + ).count() self.post_data = {} - self.post_data['map_name'] = 'test_mapbox_layer' - self.post_data['link'] = 'http://0.0.0.0:8080' + self.post_data["map_name"] = "test_mapbox_layer" + self.post_data["link"] = "http://0.0.0.0:8080" self.client.post(self.edit_url, self.post_data) - self.assertEquals(count + 1, MetaData.objects.filter( - object_id=self.xform.id, data_type='mapbox_layer').count()) + self.assertEqual( + count + 1, + MetaData.objects.filter( + object_id=self.xform.id, data_type="mapbox_layer" + ).count(), + ) def test_shows_supporting_doc_after_submit(self): name = self._add_metadata() @@ -128,7 +167,7 @@ def test_shows_supporting_doc_after_submit(self): self.assertContains(response, name) def test_shows_supporting_media_after_submit(self): - name = self._add_metadata(data_type='media') + name = self._add_metadata(data_type="media") response = self.client.get(self.url) self.assertContains(response, name) self.xform.shared = True @@ -139,23 +178,23 @@ def test_shows_supporting_media_after_submit(self): def test_shows_mapbox_layer_after_submit(self): self.post_data = {} - self.post_data['map_name'] = 'test_mapbox_layer' - self.post_data['link'] = 'http://0.0.0.0:8080' + self.post_data["map_name"] = "test_mapbox_layer" + self.post_data["link"] = "http://0.0.0.0:8080" response = self.client.post(self.edit_url, self.post_data) response = self.client.get(self.url) - self.assertContains(response, 'test_mapbox_layer') + self.assertContains(response, "test_mapbox_layer") self.xform.shared = True self.xform.save() response = self.anon.get(self.url) self.assertEqual(response.status_code, 200) - self.assertContains(response, 'test_mapbox_layer') + self.assertContains(response, "test_mapbox_layer") def test_download_supporting_doc(self): self._add_metadata() response = self.client.get(self.doc_url) self.assertEqual(response.status_code, 200) - fileName, ext = os.path.splitext(response['Content-Disposition']) - self.assertEqual(ext, '.xlsx') + fileName, ext = os.path.splitext(response["Content-Disposition"]) + self.assertEqual(ext, ".xlsx") def test_no_download_supporting_doc_for_anon(self): self._add_metadata() @@ -163,11 +202,11 @@ def test_no_download_supporting_doc_for_anon(self): self.assertEqual(response.status_code, 403) def test_download_supporting_media(self): - self._add_metadata(data_type='media') + self._add_metadata(data_type="media") response = self.client.get(self.doc_url) self.assertEqual(response.status_code, 200) - fileName, ext = os.path.splitext(response['Content-Disposition']) - self.assertEqual(ext, '.png') + fileName, ext = os.path.splitext(response["Content-Disposition"]) + self.assertEqual(ext, ".png") def test_shared_download_supporting_doc_for_anon(self): self._add_metadata() @@ -177,151 +216,191 @@ def test_shared_download_supporting_doc_for_anon(self): self.assertEqual(response.status_code, 200) def test_shared_download_supporting_media_for_anon(self): - self._add_metadata(data_type='media') + self._add_metadata(data_type="media") self.xform.shared = True self.xform.save() response = self.anon.get(self.doc_url) self.assertEqual(response.status_code, 200) def test_delete_supporting_doc(self): - count = MetaData.objects.filter(object_id=self.xform.id, - data_type='supporting_doc').count() + count = MetaData.objects.filter( + object_id=self.xform.id, data_type="supporting_doc" + ).count() self._add_metadata() - self.assertEqual(MetaData.objects.filter( - object_id=self.xform.id, - data_type='supporting_doc').count(), count + 1) - doc = MetaData.objects.filter(data_type='supporting_doc').reverse()[0] - self.delete_doc_url = reverse(delete_metadata, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - 'data_id': doc.id}) - response = self.client.get(self.delete_doc_url + '?del=true') - self.assertEqual(MetaData.objects.filter( - object_id=self.xform.id, - data_type='supporting_doc').count(), count) + self.assertEqual( + MetaData.objects.filter( + object_id=self.xform.id, data_type="supporting_doc" + ).count(), + count + 1, + ) + doc = MetaData.objects.filter(data_type="supporting_doc").reverse()[0] + self.delete_doc_url = reverse( + delete_metadata, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + "data_id": doc.id, + }, + ) + response = self.client.get(self.delete_doc_url + "?del=true") + self.assertEqual( + MetaData.objects.filter( + object_id=self.xform.id, data_type="supporting_doc" + ).count(), + count, + ) self.assertEqual(response.status_code, 302) def test_delete_supporting_media(self): count = MetaData.objects.filter( - object_id=self.xform.id, data_type='media').count() - self._add_metadata(data_type='media') - self.assertEqual(MetaData.objects.filter( - object_id=self.xform.id, data_type='media').count(), count + 1) - response = self.client.get(self.doc_url + '?del=true') - self.assertEqual(MetaData.objects.filter( - object_id=self.xform.id, data_type='media').count(), count) + object_id=self.xform.id, data_type="media" + ).count() + self._add_metadata(data_type="media") + self.assertEqual( + MetaData.objects.filter(object_id=self.xform.id, data_type="media").count(), + count + 1, + ) + response = self.client.get(self.doc_url + "?del=true") + self.assertEqual( + MetaData.objects.filter(object_id=self.xform.id, data_type="media").count(), + count, + ) self.assertEqual(response.status_code, 302) - self._add_metadata(data_type='media') - response = self.anon.get(self.doc_url + '?del=true') - self.assertEqual(MetaData.objects.filter( - object_id=self.xform.id, data_type='media').count(), count + 1) + self._add_metadata(data_type="media") + response = self.anon.get(self.doc_url + "?del=true") + self.assertEqual( + MetaData.objects.filter(object_id=self.xform.id, data_type="media").count(), + count + 1, + ) self.assertEqual(response.status_code, 403) def _add_mapbox_layer(self): # check mapbox_layer metadata count self.count = MetaData.objects.filter( - object_id=self.xform.id, data_type='mapbox_layer').count() + object_id=self.xform.id, data_type="mapbox_layer" + ).count() # add mapbox_layer metadata - post_data = {'map_name': 'test_mapbox_layer', - 'link': 'http://0.0.0.0:8080'} + post_data = {"map_name": "test_mapbox_layer", "link": "http://0.0.0.0:8080"} response = self.client.post(self.edit_url, post_data) self.assertEqual(response.status_code, 302) - self.assertEqual(MetaData.objects.filter( - object_id=self.xform.id, - data_type='mapbox_layer').count(), self.count + 1) + self.assertEqual( + MetaData.objects.filter( + object_id=self.xform.id, data_type="mapbox_layer" + ).count(), + self.count + 1, + ) def test_delete_mapbox_layer(self): self._add_mapbox_layer() # delete mapbox_layer metadata - doc = MetaData.objects.filter(data_type='mapbox_layer').reverse()[0] - self.delete_doc_url = reverse(delete_metadata, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - 'data_id': doc.id}) - response = self.client.get(self.delete_doc_url + '?map_name_del=true') - self.assertEqual(MetaData.objects.filter( - object_id=self.xform.id, - data_type='mapbox_layer').count(), self.count) + doc = MetaData.objects.filter(data_type="mapbox_layer").reverse()[0] + self.delete_doc_url = reverse( + delete_metadata, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + "data_id": doc.id, + }, + ) + response = self.client.get(self.delete_doc_url + "?map_name_del=true") + self.assertEqual( + MetaData.objects.filter( + object_id=self.xform.id, data_type="mapbox_layer" + ).count(), + self.count, + ) self.assertEqual(response.status_code, 302) def test_anon_delete_mapbox_layer(self): self._add_mapbox_layer() - doc = MetaData.objects.filter(data_type='mapbox_layer').reverse()[0] - self.delete_doc_url = reverse(delete_metadata, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - 'data_id': doc.id}) - response = self.anon.get(self.delete_doc_url + '?map_name_del=true') - self.assertEqual(MetaData.objects.filter( - object_id=self.xform.id, - data_type='mapbox_layer').count(), self.count + 1) + doc = MetaData.objects.filter(data_type="mapbox_layer").reverse()[0] + self.delete_doc_url = reverse( + delete_metadata, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + "data_id": doc.id, + }, + ) + response = self.anon.get(self.delete_doc_url + "?map_name_del=true") + self.assertEqual( + MetaData.objects.filter( + object_id=self.xform.id, data_type="mapbox_layer" + ).count(), + self.count + 1, + ) self.assertEqual(response.status_code, 302) def test_user_source_edit_updates(self): - desc = 'Snooky' - response = self.client.post(self.edit_url, {'source': desc}, - HTTP_X_REQUESTED_WITH='XMLHttpRequest') + desc = "Snooky" + response = self.client.post( + self.edit_url, {"source": desc}, HTTP_X_REQUESTED_WITH="XMLHttpRequest" + ) self.assertEqual(response.status_code, 200) self.assertEqual(MetaData.source(self.xform).data_value, desc) def test_upload_source_file(self): - self._add_metadata('source') + self._add_metadata("source") self.assertNotEqual(MetaData.source(self.xform).data_file, None) def test_upload_source_file_set_value_to_name(self): - name = self._add_metadata('source') + name = self._add_metadata("source") self.assertEqual(MetaData.source(self.xform).data_value, name) def test_upload_source_file_keep_name(self): - desc = 'Snooky' - response = self.client.post(self.edit_url, {'source': desc}, - HTTP_X_REQUESTED_WITH='XMLHttpRequest') + desc = "Snooky" + response = self.client.post( + self.edit_url, {"source": desc}, HTTP_X_REQUESTED_WITH="XMLHttpRequest" + ) self.assertEqual(response.status_code, 200) - self._add_metadata('source') + self._add_metadata("source") self.assertNotEqual(MetaData.source(self.xform).data_file, None) self.assertEqual(MetaData.source(self.xform).data_value, desc) def test_media_file_hash(self): name = "screenshot.png" media_file = os.path.join( - self.this_directory, 'fixtures', 'transportation', name) + self.this_directory, "fixtures", "transportation", name + ) content_type = ContentType.objects.get_for_model(self.xform) m = MetaData.objects.create( content_type=content_type, - data_type='media', + data_type="media", object_id=self.xform.id, data_value=name, - data_file=File(open(media_file, 'rb'), name), - data_file_type='image/png') - f = open(media_file, 'rb') - media_hash = 'md5:%s' % hashlib.md5(f.read()).hexdigest() + data_file=File(open(media_file, "rb"), name), + data_file_type="image/png", + ) + f = open(media_file, "rb") + media_hash = "md5:%s" % hashlib.md5(f.read()).hexdigest() f.close() meta_hash = m.hash self.assertEqual(meta_hash, media_hash) self.assertEqual(m.file_hash, media_hash) def test_add_media_url(self): - uri = 'https://devtrac.ona.io/fieldtrips.csv' - count = MetaData.objects.filter(data_type='media').count() - self.client.post(self.edit_url, {'media_url': uri}) - self.assertEquals(count + 1, - len(MetaData.objects.filter(data_type='media'))) + uri = "https://devtrac.ona.io/fieldtrips.csv" + count = MetaData.objects.filter(data_type="media").count() + self.client.post(self.edit_url, {"media_url": uri}) + self.assertEqual(count + 1, len(MetaData.objects.filter(data_type="media"))) def test_windows_csv_file_upload(self): - count = MetaData.objects.filter(data_type='media').count() + count = MetaData.objects.filter(data_type="media").count() media_file = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'transportation.csv') - f = InMemoryUploadedFile(open(media_file, 'rb'), - 'media', - 'transportation.csv', - 'application/octet-stream', - 2625, - None) + self.this_directory, "fixtures", "transportation", "transportation.csv" + ) + f = InMemoryUploadedFile( + open(media_file, "rb"), + "media", + "transportation.csv", + "application/octet-stream", + 2625, + None, + ) MetaData.media_upload(self.xform, f) - media_list = MetaData.objects.filter(data_type='media') + media_list = MetaData.objects.filter(data_type="media") new_count = media_list.count() self.assertEqual(count + 1, new_count) - media = media_list.get(data_value='transportation.csv') - self.assertEqual(media.data_file_type, 'text/csv') + media = media_list.get(data_value="transportation.csv") + self.assertEqual(media.data_file_type, "text/csv") diff --git a/onadata/apps/main/tests/test_http_auth.py b/onadata/apps/main/tests/test_http_auth.py index 7d747cef6f..ca7e7f440b 100644 --- a/onadata/apps/main/tests/test_http_auth.py +++ b/onadata/apps/main/tests/test_http_auth.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- +"""Test basic authentication""" from django.urls import reverse from onadata.apps.main import views @@ -5,41 +7,41 @@ class TestBasicHttpAuthentication(TestBase): + """Test basic authentication""" def setUp(self): TestBase.setUp(self) - self._create_user_and_login(username='bob', password='bob') + self._create_user_and_login(username="bob", password="bob") self._publish_transportation_form() - self.api_url = reverse(views.api, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - }) + self.api_url = reverse( + views.api, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) self._logout() def test_http_auth(self): response = self.client.get(self.api_url) self.assertEqual(response.status_code, 403) # headers with invalid user/pass - response = self.client.get(self.api_url, - **self._set_auth_headers('x', 'y')) + response = self.client.get(self.api_url, **self._set_auth_headers("x", "y")) self.assertEqual(response.status_code, 403) # headers with valid user/pass - response = self.client.get(self.api_url, - **self._set_auth_headers('bob', 'bob')) - self.assertEqual(response.status_code, 200) + response = self.client.get(self.api_url, **self._set_auth_headers("bob", "bob")) + self.assertEqual(response.status_code, 200, response.content) # Set user email self.user.email = "bob@testing_pros.com" self.user.save() # headers with valid email/pass response = self.client.get( - self.api_url, **self._set_auth_headers(self.user.email, 'bob')) - self.assertEqual(response.status_code, 200) + self.api_url, **self._set_auth_headers(self.user.email, "bob") + ) + self.assertEqual(response.status_code, 200, response.content) def test_http_auth_shared_data(self): self.xform.shared_data = True self.xform.save() response = self.anon.get(self.api_url) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 200, response.content) response = self.client.get(self.api_url) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 200, response.content) diff --git a/onadata/apps/main/tests/test_metadata.py b/onadata/apps/main/tests/test_metadata.py index f119a18fb6..e038c3db88 100644 --- a/onadata/apps/main/tests/test_metadata.py +++ b/onadata/apps/main/tests/test_metadata.py @@ -1,15 +1,23 @@ - +# -*- coding: utf-8 -*- +""" +Test MetaData model. +""" from onadata.apps.logger.models import Instance, Project, XForm -from onadata.apps.main.models.meta_data import ( - MetaData, - unique_type_for_form, - upload_to) +from onadata.apps.main.models.meta_data import MetaData, unique_type_for_form, upload_to from onadata.apps.main.tests.test_base import TestBase -from onadata.libs.utils.common_tags import GOOGLE_SHEET_DATA_TYPE,\ - GOOGLE_SHEET_ID, USER_ID, UPDATE_OR_DELETE_GOOGLE_SHEET_DATA +from onadata.libs.utils.common_tags import ( + GOOGLE_SHEET_DATA_TYPE, + GOOGLE_SHEET_ID, + UPDATE_OR_DELETE_GOOGLE_SHEET_DATA, + USER_ID, +) +# pylint: disable=too-many-public-methods class TestMetaData(TestBase): + """ + Test MetaData model. + """ def setUp(self): TestBase.setUp(self) @@ -17,55 +25,82 @@ def setUp(self): self._publish_transportation_form_and_submit_instance() def test_create_metadata(self): - count = len(MetaData.objects.filter(object_id=self.xform.id, - data_type='enketo_url')) + count = len( + MetaData.objects.filter(object_id=self.xform.id, data_type="enketo_url") + ) enketo_url = "https://dmfrm.enketo.org/webform" MetaData.enketo_url(self.xform, enketo_url) - self.assertEquals(count + 1, len(MetaData.objects.filter( - object_id=self.xform.id, data_type='enketo_url'))) + self.assertEqual( + count + 1, + len( + MetaData.objects.filter(object_id=self.xform.id, data_type="enketo_url") + ), + ) def test_create_google_sheet_metadata_object(self): - count = len(MetaData.objects.filter(object_id=self.xform.id, - data_type=GOOGLE_SHEET_DATA_TYPE)) - google_sheets_actions = ( - '{} ABC100| ' - '{} True | ' - '{} 123' - ).format(GOOGLE_SHEET_ID, UPDATE_OR_DELETE_GOOGLE_SHEET_DATA, USER_ID) + count = len( + MetaData.objects.filter( + object_id=self.xform.id, data_type=GOOGLE_SHEET_DATA_TYPE + ) + ) + google_sheets_actions = ("{} ABC100| " "{} True | " "{} 123").format( + GOOGLE_SHEET_ID, UPDATE_OR_DELETE_GOOGLE_SHEET_DATA, USER_ID + ) MetaData.set_google_sheet_details(self.xform, google_sheets_actions) # change - self.assertEquals(count + 1, MetaData.objects.filter( - object_id=self.xform.id, data_type=GOOGLE_SHEET_DATA_TYPE).count()) + self.assertEqual( + count + 1, + MetaData.objects.filter( + object_id=self.xform.id, data_type=GOOGLE_SHEET_DATA_TYPE + ).count(), + ) gsheet_details = MetaData.get_google_sheet_details(self.xform.pk) - self.assertEqual({ - GOOGLE_SHEET_ID: 'ABC100', - UPDATE_OR_DELETE_GOOGLE_SHEET_DATA: 'True', - USER_ID: '123'}, gsheet_details) + self.assertEqual( + { + GOOGLE_SHEET_ID: "ABC100", + UPDATE_OR_DELETE_GOOGLE_SHEET_DATA: "True", + USER_ID: "123", + }, + gsheet_details, + ) def test_saving_same_metadata_object_doesnt_trigger_integrity_error(self): - count = len(MetaData.objects.filter(object_id=self.xform.id, - data_type='enketo_url')) + count = len( + MetaData.objects.filter(object_id=self.xform.id, data_type="enketo_url") + ) enketo_url = "https://dmfrm.enketo.org/webform" MetaData.enketo_url(self.xform, enketo_url) count += 1 - self.assertEquals(count, len(MetaData.objects.filter( - object_id=self.xform.id, data_type='enketo_url'))) + self.assertEqual( + count, + len( + MetaData.objects.filter(object_id=self.xform.id, data_type="enketo_url") + ), + ) MetaData.enketo_url(self.xform, enketo_url) - self.assertEquals(count, len(MetaData.objects.filter( - object_id=self.xform.id, data_type='enketo_url'))) + self.assertEqual( + count, + len( + MetaData.objects.filter(object_id=self.xform.id, data_type="enketo_url") + ), + ) def test_unique_type_for_form(self): metadata = unique_type_for_form( - self.xform, data_type='enketo_url', - data_value="https://dmfrm.enketo.org/webform") + self.xform, + data_type="enketo_url", + data_value="https://dmfrm.enketo.org/webform", + ) self.assertIsInstance(metadata, MetaData) metadata_1 = unique_type_for_form( - self.xform, data_type='enketo_url', - data_value="https://dmerm.enketo.org/webform") + self.xform, + data_type="enketo_url", + data_value="https://dmerm.enketo.org/webform", + ) self.assertIsInstance(metadata_1, MetaData) self.assertNotEqual(metadata.data_value, metadata_1.data_value) @@ -77,18 +112,18 @@ def test_upload_to_with_anonymous_user(self): metadata = MetaData(data_type="media") metadata.content_object = instance filename = "filename" - self.assertEquals(upload_to(metadata, filename), - "{}/{}/{}".format(self.user.username, - 'formid-media', - filename)) + self.assertEqual( + upload_to(metadata, filename), + "{}/{}/{}".format(self.user.username, "formid-media", filename), + ) # test instance with anonymous user instance_without_user = Instance(xform=self.xform) metadata.content_object = instance_without_user - self.assertEquals(upload_to(metadata, filename), - "{}/{}/{}".format(self.xform.user.username, - 'formid-media', - filename)) + self.assertEqual( + upload_to(metadata, filename), + "{}/{}/{}".format(self.xform.user.username, "formid-media", filename), + ) def test_upload_to_with_project_and_xform_instance(self): model_instance = Project(created_by=self.user) @@ -97,17 +132,17 @@ def test_upload_to_with_project_and_xform_instance(self): filename = "filename" - self.assertEquals(upload_to(metadata, filename), - "{}/{}/{}".format(self.user.username, - 'formid-media', - filename)) + self.assertEqual( + upload_to(metadata, filename), + "{}/{}/{}".format(self.user.username, "formid-media", filename), + ) model_instance = XForm(user=self.user, created_by=self.user) metadata = MetaData(data_type="media") metadata.content_object = model_instance filename = "filename" - self.assertEquals(upload_to(metadata, filename), - "{}/{}/{}".format(self.user.username, - 'formid-media', - filename)) + self.assertEqual( + upload_to(metadata, filename), + "{}/{}/{}".format(self.user.username, "formid-media", filename), + ) diff --git a/onadata/apps/main/tests/test_process.py b/onadata/apps/main/tests/test_process.py index 6cd3497550..3cbee927b5 100644 --- a/onadata/apps/main/tests/test_process.py +++ b/onadata/apps/main/tests/test_process.py @@ -1,26 +1,30 @@ +# -*- coding: utf-8 -*- +""" +Test usage process - form publishing and export. +""" import csv import fnmatch -from io import BytesIO import json import os import re -from builtins import open from datetime import datetime from hashlib import md5 -from xml.dom import minidom, Node +from io import BytesIO +from xml.dom import Node, minidom -import pytz -import requests -import openpyxl from django.conf import settings from django.core.files.uploadedfile import UploadedFile from django.urls import reverse + +import openpyxl +import pytz +import requests from django_digest.test import Client as DigestClient -from future.utils import iteritems from mock import patch +from six import iteritems from onadata.apps.logger.models import XForm -from onadata.apps.logger.models.xform import XFORM_TITLE_LENGTH +from onadata.apps.logger.models.xform import XFORM_TITLE_LENGTH, _additional_headers from onadata.apps.logger.xform_instance_parser import clean_and_parse_xml from onadata.apps.main.models import MetaData from onadata.apps.main.tests.test_base import TestBase @@ -28,33 +32,32 @@ from onadata.libs.utils.common_tags import MONGO_STRFTIME from onadata.libs.utils.common_tools import get_response_content -uuid_regex = re.compile( - r'(.*uuid[^//]+="\')([^\']+)(\'".*)', re.DOTALL) +uuid_regex = re.compile(r'(.*uuid[^//]+="\')([^\']+)(\'".*)', re.DOTALL) class TestProcess(TestBase): - loop_str = 'loop_over_transport_types_frequency' - frequency_str = 'frequency_to_referral_facility' - ambulance_key = '%s/ambulance/%s' % (loop_str, frequency_str) - bicycle_key = '%s/bicycle/%s' % (loop_str, frequency_str) - other_key = '%s/other/%s' % (loop_str, frequency_str) - taxi_key = '%s/taxi/%s' % (loop_str, frequency_str) - transport_ambulance_key = u'transport/%s' % ambulance_key - transport_bicycle_key = u'transport/%s' % bicycle_key + """ + Test form publishing processes. + """ + + loop_str = "loop_over_transport_types_frequency" + frequency_str = "frequency_to_referral_facility" + ambulance_key = f"{loop_str}/ambulance/{frequency_str}" + bicycle_key = f"{loop_str}/bicycle/{frequency_str}" + other_key = f"{loop_str}/other/{frequency_str}" + taxi_key = f"{loop_str}/taxi/{frequency_str}" + transport_ambulance_key = f"transport/{ambulance_key}" + transport_bicycle_key = f"transport/{bicycle_key}" uuid_to_submission_times = { - '5b2cc313-fc09-437e-8149-fcd32f695d41': '2013-02-14T15:37:21', - 'f3d8dc65-91a6-4d0f-9e97-802128083390': '2013-02-14T15:37:22', - '9c6f3468-cfda-46e8-84c1-75458e72805d': '2013-02-14T15:37:23', - '9f0a1508-c3b7-4c99-be00-9b237c26bcbf': '2013-02-14T15:37:24' + "5b2cc313-fc09-437e-8149-fcd32f695d41": "2013-02-14T15:37:21", + "f3d8dc65-91a6-4d0f-9e97-802128083390": "2013-02-14T15:37:22", + "9c6f3468-cfda-46e8-84c1-75458e72805d": "2013-02-14T15:37:23", + "9f0a1508-c3b7-4c99-be00-9b237c26bcbf": "2013-02-14T15:37:24", } - def setUp(self): - super(TestProcess, self).setUp() - - def tearDown(self): - super(TestProcess, self).tearDown() - + # pylint: disable=unused-argument def test_process(self, username=None, password=None): + """Test usage process.""" self._publish_xls_file() self._check_formlist() self._download_xform() @@ -67,92 +70,131 @@ def _update_dynamic_data(self): """ Update stuff like submission time so we can compare within out fixtures """ - for (uuid, submission_time) in iteritems( - self.uuid_to_submission_times): + for (uuid, submission_time) in iteritems(self.uuid_to_submission_times): i = self.xform.instances.get(uuid=uuid) - i.date_created = pytz.timezone('UTC').localize( - datetime.strptime(submission_time, MONGO_STRFTIME)) + i.date_created = pytz.timezone("UTC").localize( + datetime.strptime(submission_time, MONGO_STRFTIME) + ) i.json = i.get_full_dict() i.save() def test_uuid_submit(self): + """Test submission with uuid included.""" self._publish_xls_file() - survey = 'transport_2011-07-25_19-05-49' + survey = "transport_2011-07-25_19-05-49" path = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'instances', survey, survey + '.xml') - with open(path) as f: - post_data = {'xml_submission_file': f, 'uuid': self.xform.uuid} - url = '/submission' + self.this_directory, + "fixtures", + "transportation", + "instances", + survey, + survey + ".xml", + ) + with open(path, encoding="utf-8") as f: + post_data = {"xml_submission_file": f, "uuid": self.xform.uuid} + url = "/submission" + # pylint: disable=attribute-defined-outside-init self.response = self.client.post(url, post_data) def test_publish_xlsx_file(self): + """Test publishing an XLSX file.""" self._publish_xlsx_file() - @patch('onadata.apps.main.forms.requests') - @patch('onadata.apps.main.forms.urlopen') - def test_google_url_upload(self, mock_urlopen, mock_requests): + @patch("onadata.apps.main.forms.requests") + def test_google_url_upload(self, mock_requests): + """Test uploading an XLSForm from a Google Docs SpreadSheet URL.""" if self._internet_on(url="http://google.com"): - xls_url = "https://docs.google.com/spreadsheet/pub?"\ + xls_url = ( + "https://docs.google.com/spreadsheet/pub?" "key=0AvhZpT7ZLAWmdDhISGhqSjBOSl9XdXd5SHZHUUE2RFE&output=xlsx" + ) pre_count = XForm.objects.count() path = os.path.join( - settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", - "transportation", "transportation.xlsx") - - xls_file = open(path, 'rb') - mock_response = requests.Response() - mock_response.status_code = 200 - mock_response.headers = { - 'content-type': ("application/vnd.openxmlformats-" - "officedocument.spreadsheetml.sheet"), - 'content-disposition': ( - 'attachment; filename="transportation.' - 'xlsx"; filename*=UTF-8\'\'transportation.xlsx') - } - mock_requests.get.return_value = mock_response - mock_urlopen.return_value = xls_file - response = self.client.post('/%s/' % self.user.username, - {'xls_url': xls_url}) - - mock_urlopen.assert_called_with(xls_url) - mock_requests.get.assert_called_with(xls_url) - # cleanup the resources - xls_file.close() - # make sure publishing the survey worked - self.assertEqual(response.status_code, 200) - self.assertEqual(XForm.objects.count(), pre_count + 1) + settings.PROJECT_ROOT, + "apps", + "main", + "tests", + "fixtures", + "transportation", + "transportation.xlsx", + ) + + with open(path, "rb") as xls_file: + mock_response = requests.Response() + mock_response.status_code = 200 + mock_response.headers = { + "content-type": ( + "application/vnd.openxmlformats-" + "officedocument.spreadsheetml.sheet" + ), + "Content-Disposition": ( + 'attachment; filename="transportation.' + "xlsx\"; filename*=UTF-8''transportation.xlsx" + ), + } + mock_requests.head.return_value = mock_response + # pylint: disable=protected-access + mock_response._content = xls_file.read() + mock_requests.get.return_value = mock_response + response = self.client.post( + f"/{self.user.username}/", {"xls_url": xls_url} + ) + + mock_requests.get.assert_called_with(xls_url) + # make sure publishing the survey worked + self.assertEqual(response.status_code, 200) + self.assertEqual(XForm.objects.count(), pre_count + 1) - @patch('onadata.apps.main.forms.urlopen') - def test_url_upload(self, mock_urlopen): + @patch("onadata.apps.main.forms.requests") + def test_url_upload(self, mock_requests): + """Test uploading an XLSForm from a URL.""" if self._internet_on(url="http://google.com"): - xls_url = 'https://ona.io/examples/forms/tutorial/form.xlsx' + xls_url = "https://ona.io/examples/forms/tutorial/form.xlsx" pre_count = XForm.objects.count() path = os.path.join( - settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", - "transportation", "transportation.xlsx") - - xls_file = open(path, 'rb') - mock_urlopen.return_value = xls_file - - response = self.client.post('/%s/' % self.user.username, - {'xls_url': xls_url}) + settings.PROJECT_ROOT, + "apps", + "main", + "tests", + "fixtures", + "transportation", + "transportation.xlsx", + ) + + # pylint: disable=consider-using-with + with open(path, "rb") as xls_file: + mock_response = requests.Response() + mock_response.status_code = 200 + mock_response.headers = { + "content-type": ( + "application/vnd.openxmlformats-" + "officedocument.spreadsheetml.sheet" + ), + "content-disposition": ( + 'attachment; filename="transportation.' + "xlsx\"; filename*=UTF-8''transportation.xlsx" + ), + } + # pylint: disable=protected-access + mock_response._content = xls_file.read() + mock_requests.get.return_value = mock_response - mock_urlopen.assert_called_with(xls_url) - # cleanup the resources - xls_file.close() + response = self.client.post( + f"/{self.user.username}/", {"xls_url": xls_url} + ) + mock_requests.get.assert_called_with(xls_url) - # make sure publishing the survey worked - self.assertEqual(response.status_code, 200) - self.assertEqual(XForm.objects.count(), pre_count + 1) + # make sure publishing the survey worked + self.assertEqual(response.status_code, 200) + self.assertEqual(XForm.objects.count(), pre_count + 1) def test_bad_url_upload(self): - xls_url = 'formhuborg/pld/forms/transportation_2011_07_25/form.xlsx' + """Test uploading an XLSForm from a badly formatted URL.""" + xls_url = "formhuborg/pld/forms/transportation_2011_07_25/form.xlsx" pre_count = XForm.objects.count() - response = self.client.post('/%s/' % self.user.username, - {'xls_url': xls_url}) + response = self.client.post(f"/{self.user.username}/", {"xls_url": xls_url}) # make sure publishing the survey worked self.assertEqual(response.status_code, 200) self.assertEqual(XForm.objects.count(), pre_count) @@ -162,40 +204,45 @@ def test_bad_url_upload(self): # containing the files you would like to test. # DO NOT CHECK IN PRIVATE XLS FILES!! def test_upload_all_xls(self): + """Test all XLSForms in online_xls folder can upload successfuly.""" root_dir = os.path.join(self.this_directory, "fixtures", "online_xls") if os.path.exists(root_dir): success = True - for root, sub_folders, filenames in os.walk(root_dir): + for root, _sub_folders, filenames in os.walk(root_dir): # ignore files that don't end in '.xlsx' - for filename in fnmatch.filter(filenames, '*.xlsx'): - success = self._publish_file(os.path.join(root, filename), - False) + for filename in fnmatch.filter(filenames, "*.xlsx"): + success = self._publish_file(os.path.join(root, filename), False) if success: # delete it so we don't have id_string conflicts if self.xform: self.xform.delete() + # pylint: disable=attribute-defined-outside-init self.xform = None - print('finished sub-folder %s' % root) + print(f"finished sub-folder {root}") self.assertEqual(success, True) + # pylint: disable=invalid-name def test_url_upload_non_dot_xls_path(self): + """Test a non .xls URL allows XLSForm upload.""" if self._internet_on(): - xls_url = 'http://formhub.org/formhub_u/forms/tutorial/form.xlsx' + xls_url = "http://formhub.org/formhub_u/forms/tutorial/form.xlsx" pre_count = XForm.objects.count() - response = self.client.post('/%s/' % self.user.username, - {'xls_url': xls_url}) + response = self.client.post(f"/{self.user.username}", {"xls_url": xls_url}) # make sure publishing the survey worked self.assertEqual(response.status_code, 200) self.assertEqual(XForm.objects.count(), pre_count + 1) + # pylint: disable=invalid-name def test_not_logged_in_cannot_upload(self): - path = os.path.join(self.this_directory, "fixtures", "transportation", - "transportation.xlsx") - if not path.startswith('/%s/' % self.user.username): + """Test anonymous user cannot upload an XLSForm.""" + path = os.path.join( + self.this_directory, "fixtures", "transportation", "transportation.xlsx" + ) + if not path.startswith(f"/{self.user.username}"): path = os.path.join(self.this_directory, path) - with open(path, 'rb') as xls_file: - post_data = {'xls_file': xls_file} - return self.client.post('/%s/' % self.user.username, post_data) + with open(path, "rb") as xls_file: + post_data = {"xls_file": xls_file} + return self.client.post(f"/{self.user.username}", post_data) def _publish_file(self, xls_path, strict=True): """ @@ -205,67 +252,68 @@ def _publish_file(self, xls_path, strict=True): TestBase._publish_xls_file(self, xls_path) # make sure publishing the survey worked if XForm.objects.count() != pre_count + 1: - print('\nPublish Failure for file: %s' % xls_path) + print(f"\nPublish Failure for file: {xls_path}") if strict: self.assertEqual(XForm.objects.count(), pre_count + 1) else: return False + # pylint: disable=attribute-defined-outside-init self.xform = list(XForm.objects.all())[-1] return True - def _publish_xls_file(self): - xls_path = os.path.join(self.this_directory, "fixtures", - "transportation", "transportation.xlsx") + def _publish_xls_file(self, path=None): + xls_path = os.path.join( + self.this_directory, "fixtures", "transportation", "transportation.xlsx" + ) self._publish_file(xls_path) self.assertEqual(self.xform.id_string, "transportation_2011_07_25") def _check_formlist(self): - url = '/%s/formList' % self.user.username + url = f"/{self.user.username}/formList" client = DigestClient() - client.set_authorization('bob', 'bob') + client.set_authorization("bob", "bob") response = client.get(url) - self.download_url = \ - 'http://testserver/%s/forms/%s/form.xml'\ - % (self.user.username, self.xform.pk) - md5_hash = md5(self.xform.xml.encode('utf-8')).hexdigest() - expected_content = """ -transportation_2011_07_25transportation_2011_07_252014111md5:%(hash)s%(download_url)s""" # noqa - expected_content = expected_content % { - 'download_url': self.download_url, - 'hash': md5_hash - } - self.assertEqual(response.content.decode('utf-8'), expected_content) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue(response.has_header('Date')) + # pylint: disable=attribute-defined-outside-init + self.download_url = ( + f"http://testserver/{self.user.username}/forms/{self.xform.pk}/form.xml" + ) + md5_hash = md5(self.xform.xml.encode("utf-8")).hexdigest() + expected_content = f""" +transportation_2011_07_25transportation_2011_07_252014111md5:{md5_hash}{self.download_url}""" # noqa + self.assertEqual(response.content.decode("utf-8"), expected_content) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("Date")) def _download_xform(self): client = DigestClient() - client.set_authorization('bob', 'bob') + client.set_authorization("bob", "bob") response = client.get(self.download_url) response_doc = minidom.parseString(response.content) - xml_path = os.path.join(self.this_directory, "fixtures", - "transportation", "transportation.xml") - with open(xml_path, 'rb') as xml_file: + xml_path = os.path.join( + self.this_directory, "fixtures", "transportation", "transportation.xml" + ) + with open(xml_path, "rb") as xml_file: expected_doc = minidom.parse(xml_file) model_node = [ - n for n in - response_doc.getElementsByTagName("h:head")[0].childNodes - if n.nodeType == Node.ELEMENT_NODE and - n.tagName == "model"][0] + n + for n in response_doc.getElementsByTagName("h:head")[0].childNodes + if n.nodeType == Node.ELEMENT_NODE and n.tagName == "model" + ][0] # check for UUID and remove - uuid_nodes = [node for node in model_node.childNodes - if node.nodeType == Node.ELEMENT_NODE and - node.getAttribute("nodeset") == - "/data/formhub/uuid"] + uuid_nodes = [ + node + for node in model_node.childNodes + if node.nodeType == Node.ELEMENT_NODE + and node.getAttribute("nodeset") == "/data/formhub/uuid" + ] self.assertEqual(len(uuid_nodes), 1) uuid_node = uuid_nodes[0] uuid_node.setAttribute("calculate", "''") - response_xml = response_doc.toxml().replace( - self.xform.version, u"201411120717") + response_xml = response_doc.toxml().replace(self.xform.version, "201411120717") # check content without UUID self.assertEqual(response_xml, expected_doc.toxml()) @@ -278,93 +326,104 @@ def _check_csv_export(self): def _check_data_dictionary(self): # test to make sure the data dictionary returns the expected headers - qs = DataDictionary.objects.filter(user=self.user) - self.assertEqual(qs.count(), 1) + queryset = DataDictionary.objects.filter(user=self.user) + self.assertEqual(queryset.count(), 1) + # pylint: disable=attribute-defined-outside-init self.data_dictionary = DataDictionary.objects.all()[0] - with open(os.path.join(self.this_directory, "fixtures", - "transportation", "headers.json")) as f: + with open( + os.path.join( + self.this_directory, "fixtures", "transportation", "headers.json" + ), + encoding="utf-8", + ) as f: expected_list = json.load(f) self.assertEqual(self.data_dictionary.get_headers(), expected_list) # test to make sure the headers in the actual csv are as expected actual_csv = self._get_csv_() - with open(os.path.join(self.this_directory, "fixtures", - "transportation", "headers_csv.json")) as f: + with open( + os.path.join( + self.this_directory, "fixtures", "transportation", "headers_csv.json" + ), + encoding="utf-8", + ) as f: expected_list = json.load(f) self.assertEqual(sorted(next(actual_csv)), sorted(expected_list)) def _check_data_for_csv_export(self): data = [ - {"available_transportation_types_to_referral_facility/ambulance": - True, - "available_transportation_types_to_referral_facility/bicycle": - True, - self.ambulance_key: "daily", - self.bicycle_key: "weekly" - }, + { + "available_transportation_types_to_referral_facility/ambulance": True, + "available_transportation_types_to_referral_facility/bicycle": True, + self.ambulance_key: "daily", + self.bicycle_key: "weekly", + }, {}, - {"available_transportation_types_to_referral_facility/ambulance": - True, - self.ambulance_key: "weekly", - }, - {"available_transportation_types_to_referral_facility/taxi": True, - "available_transportation_types_to_referral_facility/other": True, - "available_transportation_types_to_referral_facility_other": - "camel", - self.taxi_key: "daily", - self.other_key: "other", - } + { + "available_transportation_types_to_referral_facility/ambulance": True, + self.ambulance_key: "weekly", + }, + { + "available_transportation_types_to_referral_facility/taxi": True, + "available_transportation_types_to_referral_facility/other": True, + "available_transportation_types_to_referral_facility_other": "camel", + self.taxi_key: "daily", + self.other_key: "other", + }, ] for d_from_db in self.data_dictionary.get_data_for_excel(): test_dict = {} for (k, v) in iteritems(d_from_db): - if (k not in [u'_xform_id_string', u'meta/instanceID', - '_version', '_id', 'image1']) and v: - new_key = k[len('transport/'):] + if ( + k + not in [ + "_xform_id_string", + "meta/instanceID", + "_version", + "_id", + "image1", + ] + ) and v: + new_key = k[len("transport/") :] test_dict[new_key] = d_from_db[k] self.assertTrue(test_dict in data, (test_dict, data)) data.remove(test_dict) - self.assertEquals(data, []) + self.assertEqual(data, []) def _check_group_xpaths_do_not_appear_in_dicts_for_export(self): - uuid = u'uuid:f3d8dc65-91a6-4d0f-9e97-802128083390' - instance = self.xform.instances.get(uuid=uuid.split(':')[1]) + uuid = "uuid:f3d8dc65-91a6-4d0f-9e97-802128083390" + instance = self.xform.instances.get(uuid=uuid.split(":")[1]) expected_dict = { - u"transportation": { - u"meta": { - u"instanceID": uuid - }, - u"transport": { - u"loop_over_transport_types_frequency": {u"bicycle": { - u"frequency_to_referral_facility": u"weekly" - }, - u"ambulance": { - u"frequency_to_referral_facility": u"daily" - } + "transportation": { + "meta": {"instanceID": uuid}, + "transport": { + "loop_over_transport_types_frequency": { + "bicycle": {"frequency_to_referral_facility": "weekly"}, + "ambulance": {"frequency_to_referral_facility": "daily"}, }, - u"available_transportation_types_to_referral_facility": - u"ambulance bicycle", - } + "available_transportation_types_to_referral_facility": "ambulance bicycle", + }, } } self.assertEqual(instance.get_dict(flat=False), expected_dict) expected_dict = { - u"transport/available_transportation_types_to_referral_facility": - u"ambulance bicycle", - self.transport_ambulance_key: u"daily", - self.transport_bicycle_key: u"weekly", - u"_xform_id_string": u"transportation_2011_07_25", - u"_version": u"2014111", - u"meta/instanceID": uuid + "transport/available_transportation_types_to_referral_facility": "ambulance bicycle", + self.transport_ambulance_key: "daily", + self.transport_bicycle_key: "weekly", + "_xform_id_string": "transportation_2011_07_25", + "_version": "2014111", + "meta/instanceID": uuid, } self.assertEqual(instance.get_dict(), expected_dict) def _get_csv_(self): # todo: get the csv.reader to handle unicode as done here: # http://docs.python.org/library/csv.html#examples - url = reverse('csv_export', kwargs={ - 'username': self.user.username, 'id_string': self.xform.id_string}) + url = reverse( + "csv_export", + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.client.get(url) self.assertEqual(response.status_code, 200) actual_csv = get_response_content(response) @@ -372,18 +431,23 @@ def _get_csv_(self): return csv.reader(actual_lines) def _check_csv_export_first_pass(self): - url = reverse('csv_export', kwargs={ - 'username': self.user.username, 'id_string': self.xform.id_string}) + url = reverse( + "csv_export", + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.client.get(url) self.assertEqual(response.status_code, 200) test_file_path = os.path.join( - self.this_directory, "fixtures", - "transportation", "transportation.csv") + self.this_directory, "fixtures", "transportation", "transportation.csv" + ) self._test_csv_response(response, test_file_path) + # pylint: disable=too-many-locals def _check_csv_export_second_pass(self): - url = reverse('csv_export', kwargs={ - 'username': self.user.username, 'id_string': self.xform.id_string}) + url = reverse( + "csv_export", + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.client.get(url) self.assertEqual(response.status_code, 200) actual_csv = get_response_content(response) @@ -391,70 +455,90 @@ def _check_csv_export_second_pass(self): actual_csv = csv.reader(actual_lines) headers = next(actual_csv) data = [ - {"image1": "1335783522563.jpg", - 'meta/instanceID': 'uuid:5b2cc313-fc09-437e-8149-fcd32f695d41', - '_uuid': '5b2cc313-fc09-437e-8149-fcd32f695d41', - '_submission_time': '2013-02-14T15:37:21', - '_tags': '', '_notes': '', '_version': '2014111', '_duration': '', - '_submitted_by': 'bob', '_total_media': '1', '_media_count': '0', - }, - {"available_transportation_types_to_referral_facility/ambulance": - "True", - "available_transportation_types_to_referral_facility/bicycle": - "True", - self.ambulance_key: "daily", - self.bicycle_key: "weekly", - "meta/instanceID": "uuid:f3d8dc65-91a6-4d0f-9e97-802128083390", - '_uuid': 'f3d8dc65-91a6-4d0f-9e97-802128083390', - '_submission_time': '2013-02-14T15:37:22', - '_tags': '', '_notes': '', '_version': '2014111', '_duration': '', - '_submitted_by': 'bob', '_total_media': '0', '_media_count': '0', - '_media_all_received': 'True' - }, - {"available_transportation_types_to_referral_facility/ambulance": - "True", - self.ambulance_key: "weekly", - "meta/instanceID": "uuid:9c6f3468-cfda-46e8-84c1-75458e72805d", - '_uuid': '9c6f3468-cfda-46e8-84c1-75458e72805d', - '_submission_time': '2013-02-14T15:37:23', - '_tags': '', '_notes': '', '_version': '2014111', '_duration': '', - '_submitted_by': 'bob', '_total_media': '0', '_media_count': '0', - '_media_all_received': 'True' - }, - {"available_transportation_types_to_referral_facility/taxi": - "True", - "available_transportation_types_to_referral_facility/other": - "True", - "available_transportation_types_to_referral_facility_other": - "camel", - self.taxi_key: "daily", - "meta/instanceID": "uuid:9f0a1508-c3b7-4c99-be00-9b237c26bcbf", - '_uuid': '9f0a1508-c3b7-4c99-be00-9b237c26bcbf', - '_submission_time': '2013-02-14T15:37:24', - '_tags': '', '_notes': '', '_version': '2014111', '_duration': '', - '_submitted_by': 'bob', '_total_media': '0', '_media_count': '0', - '_media_all_received': 'True' - } + { + "image1": "1335783522563.jpg", + "meta/instanceID": "uuid:5b2cc313-fc09-437e-8149-fcd32f695d41", + "_uuid": "5b2cc313-fc09-437e-8149-fcd32f695d41", + "_submission_time": "2013-02-14T15:37:21", + "_tags": "", + "_notes": "", + "_version": "2014111", + "_duration": "", + "_submitted_by": "bob", + "_total_media": "1", + "_media_count": "0", + }, + { + "available_transportation_types_to_referral_facility/ambulance": "True", + "available_transportation_types_to_referral_facility/bicycle": "True", + self.ambulance_key: "daily", + self.bicycle_key: "weekly", + "meta/instanceID": "uuid:f3d8dc65-91a6-4d0f-9e97-802128083390", + "_uuid": "f3d8dc65-91a6-4d0f-9e97-802128083390", + "_submission_time": "2013-02-14T15:37:22", + "_tags": "", + "_notes": "", + "_version": "2014111", + "_duration": "", + "_submitted_by": "bob", + "_total_media": "0", + "_media_count": "0", + "_media_all_received": "True", + }, + { + "available_transportation_types_to_referral_facility/ambulance": "True", + self.ambulance_key: "weekly", + "meta/instanceID": "uuid:9c6f3468-cfda-46e8-84c1-75458e72805d", + "_uuid": "9c6f3468-cfda-46e8-84c1-75458e72805d", + "_submission_time": "2013-02-14T15:37:23", + "_tags": "", + "_notes": "", + "_version": "2014111", + "_duration": "", + "_submitted_by": "bob", + "_total_media": "0", + "_media_count": "0", + "_media_all_received": "True", + }, + { + "available_transportation_types_to_referral_facility/taxi": "True", + "available_transportation_types_to_referral_facility/other": "True", + "available_transportation_types_to_referral_facility_other": "camel", + self.taxi_key: "daily", + "meta/instanceID": "uuid:9f0a1508-c3b7-4c99-be00-9b237c26bcbf", + "_uuid": "9f0a1508-c3b7-4c99-be00-9b237c26bcbf", + "_submission_time": "2013-02-14T15:37:24", + "_tags": "", + "_notes": "", + "_version": "2014111", + "_duration": "", + "_submitted_by": "bob", + "_total_media": "0", + "_media_count": "0", + "_media_all_received": "True", + }, ] - dd = DataDictionary.objects.get(pk=self.xform.pk) - additional_headers = dd._additional_headers() + [ - '_id', '_date_modified'] + additional_headers = _additional_headers() + [ + "_id", + "_date_modified", + ] for row, expected_dict in zip(actual_csv, data): test_dict = {} - d = dict(zip(headers, row)) - for (k, v) in iteritems(d): + row_dict = dict(zip(headers, row)) + for (k, v) in iteritems(row_dict): if not (v in ["n/a", "False"] or k in additional_headers): test_dict[k] = v this_list = [] for k, v in expected_dict.items(): - if k in ['image1', 'meta/instanceID'] or k.startswith("_"): + if k in ["image1", "meta/instanceID"] or k.startswith("_"): this_list.append((k, v)) else: this_list.append(("transport/" + k, v)) self.assertEqual(test_dict, dict(this_list)) def test_xls_export_content(self): + """Test publish and export XLS content.""" self._publish_xls_file() self._make_submissions() self._update_dynamic_data() @@ -462,25 +546,31 @@ def test_xls_export_content(self): def _check_xls_export(self): xls_export_url = reverse( - 'xls_export', kwargs={'username': self.user.username, - 'id_string': self.xform.id_string}) + "xls_export", + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.client.get(xls_export_url) - expected_xls = openpyxl.open(filename=os.path.join( - self.this_directory, "fixtures", "transportation", - "transportation_export.xlsx"), data_only=True) + expected_xls = openpyxl.open( + filename=os.path.join( + self.this_directory, + "fixtures", + "transportation", + "transportation_export.xlsx", + ), + data_only=True, + ) content = get_response_content(response, decode=False) actual_xls = openpyxl.load_workbook(filename=BytesIO(content)) - actual_sheet = actual_xls.get_sheet_by_name('data') - expected_sheet = expected_xls.get_sheet_by_name('transportation') + actual_sheet = actual_xls.get_sheet_by_name("data") + expected_sheet = expected_xls.get_sheet_by_name("transportation") # check headers - self.assertEqual(list(actual_sheet.values)[0], - list(expected_sheet.values)[0]) + self.assertEqual(list(actual_sheet.values)[0], list(expected_sheet.values)[0]) # check cell data - self.assertEqual(len(list(actual_sheet.columns)), - len(list(expected_sheet.columns))) - self.assertEqual(len(list(actual_sheet.rows)), - len(list(expected_sheet.rows))) + self.assertEqual( + len(list(actual_sheet.columns)), len(list(expected_sheet.columns)) + ) + self.assertEqual(len(list(actual_sheet.rows)), len(list(expected_sheet.rows))) for i in range(1, len(list(actual_sheet.columns))): i = 1 actual_row = list(list(actual_sheet.values)[i]) @@ -492,16 +582,16 @@ def _check_xls_export(self): self.assertEqual(actual_row, expected_row) def _check_delete(self): - self.assertEquals(self.user.xforms.count(), 1) + self.assertEqual(self.user.xforms.count(), 1) self.user.xforms.all()[0].delete() - self.assertEquals(self.user.xforms.count(), 0) + self.assertEqual(self.user.xforms.count(), 0) def test_405_submission(self): - url = reverse('submissions') + url = reverse("submissions") response = self.client.get(url) - self.assertContains( - response, 'Method "GET" not allowed', status_code=405) + self.assertContains(response, 'Method "GET" not allowed', status_code=405) + # pylint: disable=invalid-name def test_publish_bad_xls_with_unicode_in_error(self): """ Publish an xls where the error has a unicode character @@ -510,40 +600,47 @@ def test_publish_bad_xls_with_unicode_in_error(self): """ self._create_user_and_login() path = os.path.join( - self.this_directory, 'fixtures', - 'form_with_unicode_in_relevant_column.xlsx') - with open(path, 'rb') as xls_file: - post_data = {'xls_file': xls_file} - response = self.client.post('/%s/' % self.user.username, post_data) + self.this_directory, "fixtures", "form_with_unicode_in_relevant_column.xlsx" + ) + with open(path, "rb") as xls_file: + post_data = {"xls_file": xls_file} + response = self.client.post(f"/{self.user.username}/", post_data) self.assertEqual(response.status_code, 200) def test_metadata_file_hash(self): + """Test a metadata file hash is generated.""" self._publish_transportation_form() - src = os.path.join(self.this_directory, "fixtures", - "transportation", "screenshot.png") - uf = UploadedFile(file=open(src, 'rb'), content_type='image/png') - count = MetaData.objects.count() - MetaData.media_upload(self.xform, uf) - # assert successful insert of new metadata record - self.assertEqual(MetaData.objects.count(), count + 1) - md = MetaData.objects.get(object_id=self.xform.id, - data_value='screenshot.png') - # assert checksum string has been generated, hash length > 1 - self.assertTrue(len(md.hash) > 16) - + src = os.path.join( + self.this_directory, "fixtures", "transportation", "screenshot.png" + ) + with open(src, "rb") as screenshot_file: + upload_file = UploadedFile(file=screenshot_file, content_type="image/png") + count = MetaData.objects.count() + MetaData.media_upload(self.xform, upload_file) + # assert successful insert of new metadata record + self.assertEqual(MetaData.objects.count(), count + 1) + metadata = MetaData.objects.get( + object_id=self.xform.id, data_value="screenshot.png" + ) + # assert checksum string has been generated, hash length > 1 + self.assertTrue(len(metadata.hash) > 16) + + # pylint: disable=invalid-name def test_uuid_injection_in_cascading_select(self): """ - Uuid is injected in the right instance for forms with cascading select + UUID is injected in the right instance for forms with cascading select """ pre_count = XForm.objects.count() xls_path = os.path.join( - self.this_directory, "fixtures", "cascading_selects", - "new_cascading_select.xlsx") - file_name, file_ext = os.path.splitext(os.path.split(xls_path)[1]) + self.this_directory, + "fixtures", + "cascading_selects", + "new_cascading_select.xlsx", + ) TestBase._publish_xls_file(self, xls_path) post_count = XForm.objects.count() self.assertEqual(post_count, pre_count + 1) - xform = XForm.objects.latest('date_created') + xform = XForm.objects.latest("date_created") # check that the uuid is within the main instance/ # the one without an id attribute @@ -551,18 +648,24 @@ def test_uuid_injection_in_cascading_select(self): # check for instance nodes that are direct children of the model node model_node = xml.getElementsByTagName("model")[0] - instance_nodes = [node for node in model_node.childNodes if - node.nodeType == Node.ELEMENT_NODE and - node.tagName.lower() == "instance" and - not node.hasAttribute("id")] + instance_nodes = [ + node + for node in model_node.childNodes + if node.nodeType == Node.ELEMENT_NODE + and node.tagName.lower() == "instance" + and not node.hasAttribute("id") + ] self.assertEqual(len(instance_nodes), 1) instance_node = instance_nodes[0] # get the first element whose id attribute is equal to our form's # id_string - form_nodes = [node for node in instance_node.childNodes if - node.nodeType == Node.ELEMENT_NODE and - node.getAttribute("id") == xform.id_string] + form_nodes = [ + node + for node in instance_node.childNodes + if node.nodeType == Node.ELEMENT_NODE + and node.getAttribute("id") == xform.id_string + ] form_node = form_nodes[0] # find the formhub node that has a uuid child node @@ -572,41 +675,49 @@ def test_uuid_injection_in_cascading_select(self): self.assertEqual(len(uuid_nodes), 1) # check for the calculate bind - calculate_bind_nodes = [node for node in model_node.childNodes if - node.nodeType == Node.ELEMENT_NODE and - node.tagName == "bind" and - node.getAttribute("nodeset") == - "/data/formhub/uuid"] + calculate_bind_nodes = [ + node + for node in model_node.childNodes + if node.nodeType == Node.ELEMENT_NODE + and node.tagName == "bind" + and node.getAttribute("nodeset") == "/data/formhub/uuid" + ] self.assertEqual(len(calculate_bind_nodes), 1) calculate_bind_node = calculate_bind_nodes[0] self.assertEqual( - calculate_bind_node.getAttribute("calculate"), "'%s'" % xform.uuid) + calculate_bind_node.getAttribute("calculate"), f"'{xform.uuid}'" + ) def test_csv_publishing(self): - csv_text = '\n'.join([ - 'survey,,', ',type,name,label', - ',text,whatsyourname,"What is your name?"', 'choices,,']) - url = reverse('user_profile', - kwargs={'username': self.user.username}) + """Test publishing a CSV XLSForm.""" + csv_text = "\n".join( + [ + "survey,,", + ",type,name,label", + ',text,whatsyourname,"What is your name?"', + "choices,,", + ] + ) + url = reverse("user_profile", kwargs={"username": self.user.username}) num_xforms = XForm.objects.count() - params = { - 'text_xls_form': csv_text - } - self.response = self.client.post(url, params) + params = {"text_xls_form": csv_text} + self.client.post(url, params) self.assertEqual(XForm.objects.count(), num_xforms + 1) + # pylint: disable=invalid-name def test_truncate_xform_title_to_255(self): + """Test the XLSForm title is truncated at 255 characters.""" self._publish_transportation_form() title = "a" * (XFORM_TITLE_LENGTH + 1) groups = re.match( - r"(.+)([^<]+)(.*)", - self.xform.xml, re.DOTALL).groups() - self.xform.xml = "{0}{1}{2}".format( - groups[0], title, groups[2]) + r"(.+)([^<]+)(.*)", self.xform.xml, re.DOTALL + ).groups() + self.xform.xml = f"{groups[0]}{title}{groups[2]}" self.xform.title = title self.xform.save() self.assertEqual(self.xform.title, "a" * XFORM_TITLE_LENGTH) + # pylint: disable=invalid-name def test_multiple_submissions_by_different_users(self): """ Two users publishing the same form breaks the CSV export. diff --git a/onadata/apps/main/tests/test_user_id_string_unique_together.py b/onadata/apps/main/tests/test_user_id_string_unique_together.py index 3f0d0e1fe9..aec4c9ed8e 100644 --- a/onadata/apps/main/tests/test_user_id_string_unique_together.py +++ b/onadata/apps/main/tests/test_user_id_string_unique_together.py @@ -19,14 +19,14 @@ def test_unique_together(self): # first time self._publish_xls_file(xls_path) - self.assertEquals(XForm.objects.count(), 1) + self.assertEqual(XForm.objects.count(), 1) # second time self.assertRaises(IntegrityError, self._publish_xls_file, xls_path) - self.assertEquals(XForm.objects.count(), 1) + self.assertEqual(XForm.objects.count(), 1) self.client.logout() # first time self._create_user_and_login(username="carl", password="carl") self._publish_xls_file(xls_path) - self.assertEquals(XForm.objects.count(), 2) + self.assertEqual(XForm.objects.count(), 2) diff --git a/onadata/apps/main/tests/test_user_settings.py b/onadata/apps/main/tests/test_user_settings.py index 0869a1e8bd..9d91863090 100644 --- a/onadata/apps/main/tests/test_user_settings.py +++ b/onadata/apps/main/tests/test_user_settings.py @@ -1,5 +1,5 @@ from django.urls import reverse -from future.utils import iteritems +from six import iteritems from onadata.apps.main.models import UserProfile from onadata.apps.main.tests.test_base import TestBase @@ -7,18 +7,18 @@ class TestUserSettings(TestBase): - def setUp(self): TestBase.setUp(self) self.settings_url = reverse( - profile_settings, kwargs={'username': self.user.username}) + profile_settings, kwargs={"username": self.user.username} + ) def test_render_user_settings(self): response = self.client.get(self.settings_url) self.assertEqual(response.status_code, 200) def test_access_user_settings_non_owner(self): - self._create_user_and_login('alice', 'alice') + self._create_user_and_login("alice", "alice") response = self.client.get(self.settings_url) self.assertEqual(response.status_code, 404) @@ -31,14 +31,14 @@ def test_show_existing_profile_data(self): def test_update_user_settings(self): post_data = { - 'name': 'Bobby', - 'organization': 'Bob Inc', - 'city': 'Bobville', - 'country': 'BB', - 'twitter': 'bobo', - 'home_page': 'bob.com', - 'require_auth': True, - 'email': 'bob@bob.com' + "name": "Bobby", + "organization": "Bob Inc", + "city": "Bobville", + "country": "BB", + "twitter": "bobo", + "home_page": "bob.com", + "require_auth": True, + "email": "bob@bob.com", } response = self.client.post(self.settings_url, post_data) self.assertEqual(response.status_code, 302) @@ -47,7 +47,7 @@ def test_update_user_settings(self): try: self.assertEqual(self.user.profile.__dict__[key], value) except KeyError as e: - if key == 'email': + if key == "email": self.assertEqual(self.user.__dict__[key], value) else: raise e diff --git a/onadata/apps/main/urls.py b/onadata/apps/main/urls.py index 8bdd6bfe6d..bb51a9941b 100644 --- a/onadata/apps/main/urls.py +++ b/onadata/apps/main/urls.py @@ -1,343 +1,528 @@ +# -*- coding: utf-8 -*- +""" +URLs path. +""" import sys import django - from django.conf import settings -from django.conf.urls import include, url, i18n +from django.conf.urls import i18n + +# enable the admin: +from django.contrib import admin from django.contrib.staticfiles import views as staticfiles_views -from django.urls import re_path +from django.urls import include, re_path from django.views.generic import RedirectView from onadata.apps import sms_support +from onadata.apps.api.urls.v1_urls import ( + BriefcaseViewset, + XFormListViewSet, + XFormSubmissionViewSet, +) from onadata.apps.api.urls.v1_urls import router as api_v1_router from onadata.apps.api.urls.v2_urls import router as api_v2_router -from onadata.apps.api.urls.v1_urls import XFormListViewSet -from onadata.apps.api.viewsets.xform_list_viewset import ( - PreviewXFormListViewSet -) -from onadata.apps.api.urls.v1_urls import XFormSubmissionViewSet -from onadata.apps.api.urls.v1_urls import BriefcaseViewset +from onadata.apps.api.viewsets.xform_list_viewset import PreviewXFormListViewSet +from onadata.apps.api.viewsets.xform_viewset import XFormViewSet from onadata.apps.logger import views as logger_views from onadata.apps.main import views as main_views -from onadata.apps.main.registration_urls import ( - urlpatterns as registration_patterns -) +from onadata.apps.main.registration_urls import urlpatterns as registration_patterns from onadata.apps.restservice import views as restservice_views from onadata.apps.sms_support import views as sms_support_views from onadata.apps.viewer import views as viewer_views -from onadata.apps.api.viewsets.xform_viewset import XFormViewSet - from onadata.libs.utils.analytics import init_analytics -# enable the admin: -from django.contrib import admin -TESTING = len(sys.argv) > 1 and sys.argv[1] == 'test' +TESTING = len(sys.argv) > 1 and sys.argv[1] == "test" admin.autodiscover() urlpatterns = [ # change Language - re_path(r'^i18n/', include(i18n)), - url('^api/v1/', include(api_v1_router.urls)), - url('^api/v2/', include(api_v2_router.urls)), + re_path(r"^i18n/", include(i18n)), + re_path("^api/v1/", include(api_v1_router.urls)), + re_path("^api/v2/", include(api_v2_router.urls)), # open id connect urls - url(r"^", include("oidc.urls")), - re_path(r'^api-docs/', - RedirectView.as_view(url=settings.STATIC_DOC, permanent=True)), - re_path(r'^api/$', - RedirectView.as_view(url=settings.STATIC_DOC, permanent=True)), - re_path(r'^api/v1$', RedirectView.as_view(url='/api/v1/', permanent=True)), - + re_path(r"^", include("oidc.urls")), + re_path( + r"^api-docs/", RedirectView.as_view(url=settings.STATIC_DOC, permanent=True) + ), + re_path(r"^api/$", RedirectView.as_view(url=settings.STATIC_DOC, permanent=True)), + re_path(r"^api/v1$", RedirectView.as_view(url="/api/v1/", permanent=True)), # django default stuff - re_path(r'^accounts/', include(registration_patterns)), - re_path(r'^admin/', admin.site.urls), - re_path(r'^admin/doc/', include('django.contrib.admindocs.urls')), - + re_path(r"^accounts/", include(registration_patterns)), + re_path(r"^admin/", admin.site.urls), + re_path(r"^admin/doc/", include("django.contrib.admindocs.urls")), # oath2_provider - re_path(r'^o/authorize/$', main_views.OnaAuthorizationView.as_view(), - name="oauth2_provider_authorize"), - re_path(r'^o/', include('oauth2_provider.urls', - namespace='oauth2_provider')), - + re_path( + r"^o/authorize/$", + main_views.OnaAuthorizationView.as_view(), + name="oauth2_provider_authorize", + ), + re_path(r"^o/", include("oauth2_provider.urls", namespace="oauth2_provider")), # main website views - re_path(r'^$', main_views.home), - re_path(r'^tutorial/$', main_views.tutorial, name='tutorial'), - re_path(r'^about-us/$', main_views.about_us, name='about-us'), - re_path(r'^getting_started/$', main_views.getting_started, - name='getting_started'), - re_path(r'^faq/$', main_views.faq, name='faq'), - re_path(r'^syntax/$', main_views.syntax, name='syntax'), - re_path(r'^privacy/$', main_views.privacy, name='privacy'), - re_path(r'^tos/$', main_views.tos, name='tos'), - re_path(r'^resources/$', main_views.resources, name='resources'), - re_path(r'^forms/$', main_views.form_gallery, name='forms_list'), - re_path(r'^forms/(?P[^/]+)$', main_views.show, name='form-show'), - re_path(r'^people/$', main_views.members_list, name='members-list'), - re_path(r'^xls2xform/$', main_views.xls2xform), - re_path(r'^support/$', main_views.support, name='support'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/stats$', - viewer_views.charts, name='form-stats'), - re_path(r'^login_redirect/$', main_views.login_redirect), - re_path(r'^attachment/$', viewer_views.attachment_url, - name='attachment_url'), - re_path(r'^attachment/(?P[^/]+)$', viewer_views.attachment_url, - name='attachment_url'), - re_path(r'^jsi18n/$', - django.views.i18n.JavaScriptCatalog.as_view( - packages=['onadata.apps.main', 'onadata.apps.viewer']), - name='javascript-catalog'), - re_path(r'^typeahead_usernames', main_views.username_list, - name='username_list'), - re_path(r'^(?P[^/]+)/$', main_views.profile, - name='user_profile'), - re_path(r'^(?P[^/]+)/profile$', main_views.public_profile, - name='public_profile'), - re_path(r'^(?P[^/]+)/settings', main_views.profile_settings, - name='profile-settings'), - re_path(r'^(?P[^/]+)/cloneform$', main_views.clone_xlsform, - name='clone-xlsform'), - re_path(r'^(?P[^/]+)/activity$', main_views.activity, - name='activity'), - re_path(r'^(?P[^/]+)/activity/api$', main_views.activity_api, - name='activity-api'), - re_path(r'^activity/fields$', main_views.activity_fields, - name='activity-fields'), - re_path(r'^(?P[^/]+)/api-token$', main_views.api_token, - name='api-token'), - + re_path(r"^$", main_views.home), + re_path(r"^tutorial/$", main_views.tutorial, name="tutorial"), + re_path(r"^about-us/$", main_views.about_us, name="about-us"), + re_path(r"^getting_started/$", main_views.getting_started, name="getting_started"), + re_path(r"^faq/$", main_views.faq, name="faq"), + re_path(r"^syntax/$", main_views.syntax, name="syntax"), + re_path(r"^privacy/$", main_views.privacy, name="privacy"), + re_path(r"^tos/$", main_views.tos, name="tos"), + re_path(r"^resources/$", main_views.resources, name="resources"), + re_path(r"^forms/$", main_views.form_gallery, name="forms_list"), + re_path(r"^forms/(?P[^/]+)$", main_views.show, name="form-show"), + re_path(r"^people/$", main_views.members_list, name="members-list"), + re_path(r"^xls2xform/$", main_views.xls2xform), + re_path(r"^support/$", main_views.support, name="support"), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/stats$", + viewer_views.charts, + name="form-stats", + ), + re_path(r"^login_redirect/$", main_views.login_redirect), + re_path(r"^attachment/$", viewer_views.attachment_url, name="attachment_url"), + re_path( + r"^attachment/(?P[^/]+)$", + viewer_views.attachment_url, + name="attachment_url", + ), + re_path( + r"^jsi18n/$", + django.views.i18n.JavaScriptCatalog.as_view( + packages=["onadata.apps.main", "onadata.apps.viewer"] + ), + name="javascript-catalog", + ), + re_path(r"^typeahead_usernames", main_views.username_list, name="username_list"), + re_path(r"^(?P[^/]+)/$", main_views.profile, name="user_profile"), + re_path( + r"^(?P[^/]+)/profile$", + main_views.public_profile, + name="public_profile", + ), + re_path( + r"^(?P[^/]+)/settings", + main_views.profile_settings, + name="profile-settings", + ), + re_path( + r"^(?P[^/]+)/cloneform$", + main_views.clone_xlsform, + name="clone-xlsform", + ), + re_path(r"^(?P[^/]+)/activity$", main_views.activity, name="activity"), + re_path( + r"^(?P[^/]+)/activity/api$", + main_views.activity_api, + name="activity-api", + ), + re_path(r"^activity/fields$", main_views.activity_fields, name="activity-fields"), + re_path(r"^(?P[^/]+)/api-token$", main_views.api_token, name="api-token"), # form specific - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)$', - main_views.show, - name='form-show'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/qrcode$', - main_views.qrcode, name='get_qrcode'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/api$', - main_views.api, name='mongo_view_api'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/public_api$', - main_views.public_api, name='public_api'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/delete_data$', - main_views.delete_data, name='delete_data'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/edit$', - main_views.edit, name='xform-edit'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/perms$', - main_views.set_perm, name='set-xform-permissions'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/photos', - main_views.form_photos, name='form-photos'), - - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/doc/(?P\d+)', # noqa - main_views.download_metadata, name='download-metadata'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/delete-doc/(?P\d+)', main_views.delete_metadata, # noqa - name='delete-metadata'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/formid-media/(?P\d+)', main_views.download_media_data, # noqa - name='download-media-data'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/addservice$', - restservice_views.add_service, name='add_restservice'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/delservice$', - restservice_views.delete_service, - name='delete_restservice'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/update$', - main_views.update_xform, name='update-xform'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/preview$', - main_views.enketo_preview, name='enketo-preview'), - + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)$", + main_views.show, + name="form-show", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/qrcode$", + main_views.qrcode, + name="get_qrcode", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/api$", + main_views.api, + name="mongo_view_api", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/public_api$", + main_views.public_api, + name="public_api", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/delete_data$", + main_views.delete_data, + name="delete_data", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/edit$", + main_views.edit, + name="xform-edit", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/perms$", + main_views.set_perm, + name="set-xform-permissions", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/photos", + main_views.form_photos, + name="form-photos", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/doc/(?P\d+)", # noqa + main_views.download_metadata, + name="download-metadata", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/delete-doc/(?P\d+)", + main_views.delete_metadata, # noqa + name="delete-metadata", + ), + re_path( + # pylint: disable=line-too-long + r"^(?P[^/]+)/forms/(?P[^/]+)/formid-media/(?P\d+)", + main_views.download_media_data, # noqa + name="download-media-data", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/addservice$", + restservice_views.add_service, + name="add_restservice", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/delservice$", + restservice_views.delete_service, + name="delete_restservice", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/update$", + main_views.update_xform, + name="update-xform", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/preview$", + main_views.enketo_preview, + name="enketo-preview", + ), # briefcase api urls - re_path(r'^(?P\w+)/view/submissionList$', - BriefcaseViewset.as_view({'get': 'list', 'head': 'list'}), - name='view-submission-list'), - re_path(r'^(?P\w+)/view/downloadSubmission$', - BriefcaseViewset.as_view({'get': 'retrieve', 'head': 'retrieve'}), - name='view-download-submission'), - re_path(r'^(?P\w+)/formUpload$', - BriefcaseViewset.as_view({'post': 'create', 'head': 'create'}), - name='form-upload'), - re_path(r'^(?P\w+)/upload$', - BriefcaseViewset.as_view({'post': 'create', 'head': 'create'}), - name='upload'), - + re_path( + r"^(?P\w+)/view/submissionList$", + BriefcaseViewset.as_view({"get": "list", "head": "list"}), + name="view-submission-list", + ), + re_path( + r"^(?P\w+)/view/downloadSubmission$", + BriefcaseViewset.as_view({"get": "retrieve", "head": "retrieve"}), + name="view-download-submission", + ), + re_path( + r"^(?P\w+)/formUpload$", + BriefcaseViewset.as_view({"post": "create", "head": "create"}), + name="form-upload", + ), + re_path( + r"^(?P\w+)/upload$", + BriefcaseViewset.as_view({"post": "create", "head": "create"}), + name="upload", + ), # exporting stuff - re_path(r'^(?P\w+)/forms/(?P[^/]+)/data\.csv$', - viewer_views.data_export, name='csv_export', - kwargs={'export_type': 'csv'}), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/data\.xls', - viewer_views.data_export, name='xls_export', - kwargs={'export_type': 'xls'}), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/data\.csv.zip', - viewer_views.data_export, name='csv_zip_export', - kwargs={'export_type': 'csv_zip'}), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/data\.sav.zip', - viewer_views.data_export, name='sav_zip_export', - kwargs={'export_type': 'sav_zip'}), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/data\.kml$', - viewer_views.kml_export, name='kml-export'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/data\.zip', - viewer_views.zip_export), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/gdocs$', - viewer_views.google_xls_export), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/map_embed', - viewer_views.map_embed_view), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/map', - viewer_views.map_view, name='map-view'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/instance', - viewer_views.instance, name='submission-instance'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/enter-data', - logger_views.enter_data, name='enter_data'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/add-submission-with', # noqa - viewer_views.add_submission_with, - name='add_submission_with'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/thank_you_submission', # noqa - viewer_views.thank_you_submission, - name='thank_you_submission'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/edit-data/(?P\d+)$', # noqa - logger_views.edit_data, name='edit_data'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/view-data', - viewer_views.data_view, name='data-view'), - re_path(r'^(?P\w+)/exports/(?P[^/]+)/(?P\w+)/new$', # noqa - viewer_views.create_export, name='new-export'), - re_path(r'^(?P\w+)/exports/(?P[^/]+)/(?P\w+)' # noqa - '/delete$', viewer_views.delete_export, name='delete-export'), - re_path(r'^(?P\w+)/exports/(?P[^/]+)/(?P\w+)' # noqa - '/progress$', viewer_views.export_progress, - name='export-progress'), - re_path(r'^(?P\w+)/exports/(?P[^/]+)/(?P\w+)' # noqa - '/$', viewer_views.export_list, name='export-list'), - re_path(r'^(?P\w+)/exports/(?P[^/]+)/(?P\w+)' # noqa - '/(?P[^/]+)$', - viewer_views.export_download, name='export-download'), - + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/data\.csv$", + viewer_views.data_export, + name="csv_export", + kwargs={"export_type": "csv"}, + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/data\.xls", + viewer_views.data_export, + name="xls_export", + kwargs={"export_type": "xls"}, + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/data\.csv.zip", + viewer_views.data_export, + name="csv_zip_export", + kwargs={"export_type": "csv_zip"}, + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/data\.sav.zip", + viewer_views.data_export, + name="sav_zip_export", + kwargs={"export_type": "sav_zip"}, + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/data\.kml$", + viewer_views.kml_export, + name="kml-export", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/data\.zip", + viewer_views.zip_export, + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/gdocs$", + viewer_views.google_xls_export, + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/map_embed", + viewer_views.map_embed_view, + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/map", + viewer_views.map_view, + name="map-view", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/instance", + viewer_views.instance, + name="submission-instance", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/enter-data", + logger_views.enter_data, + name="enter_data", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/add-submission-with", # noqa + viewer_views.add_submission_with, + name="add_submission_with", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/thank_you_submission", # noqa + viewer_views.thank_you_submission, + name="thank_you_submission", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/edit-data/(?P\d+)$", # noqa + logger_views.edit_data, + name="edit_data", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/view-data", + viewer_views.data_view, + name="data-view", + ), + re_path( + r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)/new$", # noqa + viewer_views.create_export, + name="new-export", + ), + re_path( + r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)" # noqa + "/delete$", + viewer_views.delete_export, + name="delete-export", + ), + re_path( + r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)" # noqa + "/progress$", + viewer_views.export_progress, + name="export-progress", + ), + re_path( + r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)" # noqa + "/$", + viewer_views.export_list, + name="export-list", + ), + re_path( + r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)" # noqa + "/(?P[^/]+)$", + viewer_views.export_download, + name="export-download", + ), # xform versions urls - re_path(r'^api/v1/forms/(?P[^/.]+)/versions/(?P[^/.]+)$', # noqa - XFormViewSet.as_view({'get': 'versions'}), - name='form-version-detail'), - re_path(r'^api/v1/forms/(?P[^/.]+)/versions/(?P[^/.]+)\.(?P[a-z0-9]+)/?$', # noqa - XFormViewSet.as_view({'get': 'versions'}), - name='form-version-detail'), - + re_path( + r"^api/v1/forms/(?P[^/.]+)/versions/(?P[^/.]+)$", # noqa + XFormViewSet.as_view({"get": "versions"}), + name="form-version-detail", + ), + re_path( + # pylint: disable=line-too-long + r"^api/v1/forms/(?P[^/.]+)/versions/(?P[^/.]+)\.(?P[a-z0-9]+)/?$", # noqa + XFormViewSet.as_view({"get": "versions"}), + name="form-version-detail", + ), # odk data urls - re_path(r'^submission$', - XFormSubmissionViewSet.as_view( - {'post': 'create', 'head': 'create'}), - name='submissions'), - re_path(r'^formList$', - XFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), - re_path(r'^(?P\w+)/formList$', - XFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), - re_path(r'^enketo/(?P\w+)/formList$', - XFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), - re_path(r'^enketo-preview/(?P\w+)/formList$', - PreviewXFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), - re_path(r'^(?P\w+)/(?P\d+)/formList$', - XFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), - re_path(r'^preview/(?P\w+)/(?P\d+)/formList$', - PreviewXFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), - re_path(r'^preview/(?P\w+)/formList$', - PreviewXFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), - re_path(r'^(?P\w+)/xformsManifest/(?P[\d+^/]+)$', - XFormListViewSet.as_view({'get': 'manifest', 'head': 'manifest'}), - name='manifest-url'), - re_path(r'^xformsManifest/(?P[\d+^/]+)$', - XFormListViewSet.as_view({'get': 'manifest', 'head': 'manifest'}), - name='manifest-url'), - re_path(r'^(?P\w+)/xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)$', # noqa - XFormListViewSet.as_view({'get': 'media', 'head': 'media'}), - name='xform-media'), - re_path(r'^(?P\w+)/xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)\.(?P([a-z]|[0-9])*)$',# noqa - XFormListViewSet.as_view({'get': 'media', 'head': 'media'}), - name='xform-media'), - re_path(r'^xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)$', - XFormListViewSet.as_view({'get': 'media', 'head': 'media'}), - name='xform-media'), - re_path(r'^xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)\.' - '(?P([a-z]|[0-9])*)$', - XFormListViewSet.as_view({'get': 'media', 'head': 'media'}), - name='xform-media'), - re_path(r'^(?P\w+)/submission$', - XFormSubmissionViewSet.as_view( - {'post': 'create', 'head': 'create'}), - name='submissions'), - re_path(r'^enketo/(?P\w+)/submission$', - XFormSubmissionViewSet.as_view( - {'post': 'create', 'head': 'create'}), - name='submissions'), - re_path(r'^(?P\w+)/(?P\d+)/submission$', - XFormSubmissionViewSet.as_view( - {'post': 'create', 'head': 'create'}), - name='submissions'), - re_path(r'^(?P\w+)/bulk-submission$', - logger_views.bulksubmission), - re_path(r'^(?P\w+)/bulk-submission-form$', - logger_views.bulksubmission_form), - re_path(r'^(?P\w+)/forms/(?P[\d+^/]+)/form\.xml$', - XFormListViewSet.as_view({'get': 'retrieve', 'head': 'retrieve'}), - name='download_xform'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/form\.xml$', - logger_views.download_xform, name='download_xform'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/form\.xls$', - logger_views.download_xlsform, - name='download_xlsform'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/form\.json', - logger_views.download_jsonform, - name='download_jsonform'), - re_path(r'^(?P\w+)/delete/(?P[^/]+)/$', - logger_views.delete_xform, name='delete-xform'), - re_path(r'^(?P\w+)/(?P[^/]+)/toggle_downloadable/$', - logger_views.toggle_downloadable, name='toggle-downloadable'), - + re_path( + r"^submission$", + XFormSubmissionViewSet.as_view({"post": "create", "head": "create"}), + name="submissions", + ), + re_path( + r"^formList$", + XFormListViewSet.as_view({"get": "list", "head": "list"}), + name="form-list", + ), + re_path( + r"^(?P\w+)/formList$", + XFormListViewSet.as_view({"get": "list", "head": "list"}), + name="form-list", + ), + re_path( + r"^enketo/(?P\w+)/formList$", + XFormListViewSet.as_view({"get": "list", "head": "list"}), + name="form-list", + ), + re_path( + r"^enketo-preview/(?P\w+)/formList$", + PreviewXFormListViewSet.as_view({"get": "list", "head": "list"}), + name="form-list", + ), + re_path( + r"^(?P\w+)/(?P\d+)/formList$", + XFormListViewSet.as_view({"get": "list", "head": "list"}), + name="form-list", + ), + re_path( + r"^preview/(?P\w+)/(?P\d+)/formList$", + PreviewXFormListViewSet.as_view({"get": "list", "head": "list"}), + name="form-list", + ), + re_path( + r"^preview/(?P\w+)/formList$", + PreviewXFormListViewSet.as_view({"get": "list", "head": "list"}), + name="form-list", + ), + re_path( + r"^(?P\w+)/xformsManifest/(?P[\d+^/]+)$", + XFormListViewSet.as_view({"get": "manifest", "head": "manifest"}), + name="manifest-url", + ), + re_path( + r"^xformsManifest/(?P[\d+^/]+)$", + XFormListViewSet.as_view({"get": "manifest", "head": "manifest"}), + name="manifest-url", + ), + re_path( + r"^(?P\w+)/xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)$", # noqa + XFormListViewSet.as_view({"get": "media", "head": "media"}), + name="xform-media", + ), + re_path( + # pylint: disable=line-too-long + r"^(?P\w+)/xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)\.(?P([a-z]|[0-9])*)$", # noqa + XFormListViewSet.as_view({"get": "media", "head": "media"}), + name="xform-media", + ), + re_path( + r"^xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)$", + XFormListViewSet.as_view({"get": "media", "head": "media"}), + name="xform-media", + ), + re_path( + r"^xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)\." + "(?P([a-z]|[0-9])*)$", + XFormListViewSet.as_view({"get": "media", "head": "media"}), + name="xform-media", + ), + re_path( + r"^(?P\w+)/submission$", + XFormSubmissionViewSet.as_view({"post": "create", "head": "create"}), + name="submissions", + ), + re_path( + r"^enketo/(?P\w+)/submission$", + XFormSubmissionViewSet.as_view({"post": "create", "head": "create"}), + name="submissions", + ), + re_path( + r"^(?P\w+)/(?P\d+)/submission$", + XFormSubmissionViewSet.as_view({"post": "create", "head": "create"}), + name="submissions", + ), + re_path(r"^(?P\w+)/bulk-submission$", logger_views.bulksubmission), + re_path( + r"^(?P\w+)/bulk-submission-form$", logger_views.bulksubmission_form + ), + re_path( + r"^(?P\w+)/forms/(?P[\d+^/]+)/form\.xml$", + XFormListViewSet.as_view({"get": "retrieve", "head": "retrieve"}), + name="download_xform", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/form\.xml$", + logger_views.download_xform, + name="download_xform", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/form\.xls$", + logger_views.download_xlsform, + name="download_xlsform", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/form\.json", + logger_views.download_jsonform, + name="download_jsonform", + ), + re_path( + r"^(?P\w+)/delete/(?P[^/]+)/$", + logger_views.delete_xform, + name="delete-xform", + ), + re_path( + r"^(?P\w+)/(?P[^/]+)/toggle_downloadable/$", + logger_views.toggle_downloadable, + name="toggle-downloadable", + ), # SMS support - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/sms_submission/(?P[a-z]+)/?$', # noqa - sms_support.providers.import_submission_for_form, - name='sms_submission_form_api'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/sms_submission$', - sms_support_views.import_submission_for_form, - name='sms_submission_form'), - re_path(r'^(?P[^/]+)/sms_submission/(?P[a-z]+)/?$', - sms_support.providers.import_submission, - name='sms_submission_api'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/sms_multiple_submissions$', # noqa - sms_support_views.import_multiple_submissions_for_form, - name='sms_submissions_form'), - re_path(r'^(?P[^/]+)/sms_multiple_submissions$', - sms_support_views.import_multiple_submissions, - name='sms_submissions'), - re_path(r'^(?P[^/]+)/sms_submission$', - sms_support_views.import_submission, name='sms_submission'), - + re_path( + # pylint: disable=line-too-long + r"^(?P[^/]+)/forms/(?P[^/]+)/sms_submission/(?P[a-z]+)/?$", # noqa + sms_support.providers.import_submission_for_form, + name="sms_submission_form_api", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/sms_submission$", + sms_support_views.import_submission_for_form, + name="sms_submission_form", + ), + re_path( + r"^(?P[^/]+)/sms_submission/(?P[a-z]+)/?$", + sms_support.providers.import_submission, + name="sms_submission_api", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/sms_multiple_submissions$", # noqa + sms_support_views.import_multiple_submissions_for_form, + name="sms_submissions_form", + ), + re_path( + r"^(?P[^/]+)/sms_multiple_submissions$", + sms_support_views.import_multiple_submissions, + name="sms_submissions", + ), + re_path( + r"^(?P[^/]+)/sms_submission$", + sms_support_views.import_submission, + name="sms_submission", + ), # Stats tables - re_path(r'^(?P\w+)/forms/(?P[^/]+)/tables', - viewer_views.stats_tables, name='stats-tables'), - + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/tables", + viewer_views.stats_tables, + name="stats-tables", + ), # static media - re_path(r'^media/(?P.*)$', django.views.static.serve, - {'document_root': settings.MEDIA_ROOT}), - re_path(r'^favicon\.ico', - RedirectView.as_view(url='/static/images/favicon.ico', - permanent=True)), - re_path(r'^static/(?P.*)$', staticfiles_views.serve), - + re_path( + r"^media/(?P.*)$", + django.views.static.serve, + {"document_root": settings.MEDIA_ROOT}, + ), + re_path( + r"^favicon\.ico", + RedirectView.as_view(url="/static/images/favicon.ico", permanent=True), + ), + re_path(r"^static/(?P.*)$", staticfiles_views.serve), # Health status - re_path(r'^status$', main_views.service_health) + re_path(r"^status$", main_views.service_health), ] -CUSTOM_URLS = getattr(settings, 'CUSTOM_MAIN_URLS', None) +CUSTOM_URLS = getattr(settings, "CUSTOM_MAIN_URLS", None) if CUSTOM_URLS: for url_module in CUSTOM_URLS: - urlpatterns.append(re_path(r'^', include(url_module))) + urlpatterns.append(re_path(r"^", include(url_module))) -if (settings.DEBUG or TESTING) and 'debug_toolbar' in settings.INSTALLED_APPS: +if (settings.DEBUG or TESTING) and "debug_toolbar" in settings.INSTALLED_APPS: try: import debug_toolbar except ImportError: pass else: urlpatterns += [ - re_path(r'^__debug__/', include(debug_toolbar.urls)), + re_path(r"^__debug__/", include(debug_toolbar.urls)), ] init_analytics() diff --git a/onadata/apps/main/views.py b/onadata/apps/main/views.py index 418d4dc71a..990fc2f6f0 100644 --- a/onadata/apps/main/views.py +++ b/onadata/apps/main/views.py @@ -6,25 +6,32 @@ import json import os from datetime import datetime +from http import HTTPStatus -from bson import json_util from django.conf import settings from django.contrib import messages +from django.contrib.auth import get_user_model from django.contrib.auth.decorators import login_required -from django.contrib.auth.models import User from django.core.cache import cache from django.core.files.storage import default_storage, get_storage_class -from django.db import IntegrityError, OperationalError -from django.http import (HttpResponse, HttpResponseBadRequest, - HttpResponseForbidden, HttpResponseNotFound, - HttpResponseRedirect, HttpResponseServerError) +from django.db import IntegrityError +from django.http import ( + HttpResponse, + HttpResponseBadRequest, + HttpResponseForbidden, + HttpResponseNotFound, + HttpResponseRedirect, + JsonResponse, +) from django.shortcuts import get_object_or_404, render -from django.template import RequestContext, loader +from django.template import loader from django.urls import reverse -from django.utils.translation import ugettext as _ from django.utils.html import conditional_escape -from django.views.decorators.http import (require_GET, require_http_methods, - require_POST) +from django.utils.translation import gettext as _ +from django.views.decorators.http import require_GET, require_http_methods, require_POST + +from bson import json_util +from bson.objectid import ObjectId from guardian.shortcuts import assign_perm, remove_perm from oauth2_provider.views.base import AuthorizationView from rest_framework.authtoken.models import Token @@ -33,54 +40,73 @@ from onadata.apps.logger.models.xform import get_forms_shared_with_user from onadata.apps.logger.views import enter_data from onadata.apps.logger.xform_instance_parser import XLSFormError -from onadata.apps.main.forms import (ActivateSMSSupportForm, DataLicenseForm, - ExternalExportForm, FormLicenseForm, - MapboxLayerForm, MediaForm, - PermissionForm, QuickConverter, - QuickConverterFile, QuickConverterURL, - SourceForm, SupportDocForm, - UserProfileForm) +from onadata.apps.main.forms import ( + ActivateSMSSupportForm, + DataLicenseForm, + ExternalExportForm, + FormLicenseForm, + MapboxLayerForm, + MediaForm, + PermissionForm, + QuickConverter, + QuickConverterFile, + QuickConverterURL, + SourceForm, + SupportDocForm, + UserProfileForm, +) from onadata.apps.main.models import AuditLog, MetaData, UserProfile from onadata.apps.sms_support.autodoc import get_autodoc_for from onadata.apps.sms_support.providers import providers_doc -from onadata.apps.sms_support.tools import (check_form_sms_compatibility, - is_sms_related) -from onadata.apps.viewer.models.data_dictionary import (DataDictionary, - upload_to) -from onadata.apps.viewer.models.parsed_instance import (DATETIME_FORMAT, - query_data) +from onadata.apps.sms_support.tools import check_form_sms_compatibility, is_sms_related +from onadata.apps.viewer.models.data_dictionary import DataDictionary, upload_to +from onadata.apps.viewer.models.parsed_instance import DATETIME_FORMAT, query_data from onadata.apps.viewer.views import attachment_url from onadata.libs.exceptions import EnketoError from onadata.libs.utils.decorators import is_owner from onadata.libs.utils.export_tools import upload_template_for_external_export from onadata.libs.utils.log import Actions, audit_log -from onadata.libs.utils.logger_tools import (publish_form, - response_with_mimetype_and_name) +from onadata.libs.utils.logger_tools import ( + publish_form, + response_with_mimetype_and_name, +) from onadata.libs.utils.qrcode import generate_qrcode -from onadata.libs.utils.user_auth import (add_cors_headers, check_and_set_user, - check_and_set_user_and_form, - get_user_default_project, - get_xform_and_perms, - get_xform_users_with_perms, - has_permission, - helper_auth_helper, set_profile_data) -from onadata.libs.utils.viewer_tools import (get_enketo_urls, get_form) +from onadata.libs.utils.user_auth import ( + add_cors_headers, + check_and_set_user, + check_and_set_user_and_form, + get_user_default_project, + get_xform_and_perms, + get_xform_users_with_perms, + has_permission, + helper_auth_helper, + set_profile_data, +) +from onadata.libs.utils.viewer_tools import get_enketo_urls, get_form + +# pylint: disable=invalid-name +User = get_user_model() def home(request): + """Default landing view.""" if request.user.username: return HttpResponseRedirect( - reverse(profile, kwargs={'username': request.user.username})) + reverse(profile, kwargs={"username": request.user.username}) + ) - return render(request, 'home.html') + return render(request, "home.html") @login_required def login_redirect(request): + """Redirects a user to their profile page on successful login.""" return HttpResponseRedirect( - reverse(profile, kwargs={'username': request.user.username})) + reverse(profile, kwargs={"username": request.user.username}) + ) +# pylint: disable=unused-argument @require_POST @login_required def clone_xlsform(request, username): @@ -89,59 +115,66 @@ def clone_xlsform(request, username): Eliminates the need to download Excel File and upload again. """ to_username = request.user.username - message = {'type': None, 'text': '....'} + message = {"type": None, "text": "...."} message_list = [] def set_form(): - form_owner = request.POST.get('username') - id_string = request.POST.get('id_string') - xform = XForm.objects.get(user__username__iexact=form_owner, - id_string__iexact=id_string, - deleted_at__isnull=True) - if len(id_string) > 0 and id_string[0].isdigit(): - id_string = '_' + id_string + """Publishes the XLSForm creating a DataDictionary object.""" + form_owner = request.POST.get("username") + id_string = request.POST.get("id_string") + xform = XForm.objects.get( + user__username__iexact=form_owner, + id_string__iexact=id_string, + deleted_at__isnull=True, + ) + if id_string and id_string[0].isdigit(): + id_string = "_" + id_string path = xform.xls.name if default_storage.exists(path): project = get_user_default_project(request.user) - xls_file = upload_to(None, '%s%s.xlsx' % ( - id_string, XForm.CLONED_SUFFIX), to_username) + xls_file = upload_to( + None, f"{id_string}{XForm.CLONED_SUFFIX}.xlsx", to_username + ) xls_data = default_storage.open(path) xls_file = default_storage.save(xls_file, xls_data) survey = DataDictionary.objects.create( - user=request.user, - xls=xls_file, - project=project + user=request.user, xls=xls_file, project=project ).survey # log to cloner's account audit = {} audit_log( - Actions.FORM_CLONED, request.user, request.user, - _("Cloned form '%(id_string)s'.") % - { - 'id_string': survey.id_string, - }, audit, request) + Actions.FORM_CLONED, + request.user, + request.user, + _(f"Cloned form '{survey.id_string}'."), + audit, + request, + ) clone_form_url = reverse( - show, kwargs={ - 'username': to_username, - 'id_string': xform.id_string + XForm.CLONED_SUFFIX}) + show, + kwargs={ + "username": to_username, + "id_string": xform.id_string + XForm.CLONED_SUFFIX, + }, + ) + profile_url = reverse(profile, kwargs={"username": to_username}) + profile_url_link = f'profile.' + form_url_link = f'{survey.id_string} ' return { - 'type': 'alert-success', - 'text': _( - u'Successfully cloned to %(form_url)s into your ' - u'%(profile_url)s') % { - 'form_url': u'%(id_string)s ' % { - 'id_string': survey.id_string, - 'url': clone_form_url - }, - 'profile_url': u'profile.' % reverse( - profile, kwargs={'username': to_username})}} + "type": "alert-success", + "text": _( + f"Successfully cloned to {form_url_link} into your " + f"{profile_url_link}" + ), + } + return {} form_result = publish_form(set_form) - if form_result['type'] == 'alert-success': + if form_result["type"] == "alert-success": # comment the following condition (and else) # when we want to enable sms check for all. # until then, it checks if form barely related to sms - if is_sms_related(form_result.get('form_o')): + if is_sms_related(form_result.get("form_o")): form_result_sms = check_form_sms_compatibility(form_result) message_list = [form_result, form_result_sms] else: @@ -149,69 +182,73 @@ def set_form(): else: message = form_result - context = RequestContext(request, { - 'message': message, 'message_list': message_list}) + context = {"message": message, "message_list": message_list} if request.is_ajax(): - res = loader.render_to_string( - 'message.html', - context_instance=context - ).replace("'", r"\'").replace('\n', '') + res = ( + loader.render_to_string("message.html", context=context, request=request) + .replace("'", r"\'") + .replace("\n", "") + ) - return HttpResponse( - "$('#mfeedback').html('%s').show();" % res) - else: - return HttpResponse(message['text']) + return HttpResponse(f"$('#mfeedback').html('{res}').show();") + return HttpResponse(message["text"]) + +# pylint: disable=too-many-locals def profile(request, username): + """Show user profiles page view.""" content_user = get_object_or_404(User, username__iexact=username) form = QuickConverter() - data = {'form': form} + data = {"form": form} # xlsform submission... - if request.method == 'POST' and request.user.is_authenticated: + if request.method == "POST" and request.user.is_authenticated: + def set_form(): + """Publishes the XLSForm.""" form = QuickConverter(request.POST, request.FILES) survey = form.publish(request.user).survey audit = {} audit_log( - Actions.FORM_PUBLISHED, request.user, content_user, - _("Published form '%(id_string)s'.") % - { - 'id_string': survey.id_string, - }, audit, request) + Actions.FORM_PUBLISHED, + request.user, + content_user, + _(f"Published form '{survey.id_string}'."), + audit, + request, + ) enketo_webform_url = reverse( - enter_data, - kwargs={'username': username, 'id_string': survey.id_string} + enter_data, kwargs={"username": username, "id_string": survey.id_string} ) return { - 'type': 'alert-success', - 'preview_url': reverse(enketo_preview, kwargs={ - 'username': username, - 'id_string': survey.id_string - }), - 'text': _(u'Successfully published %(form_id)s.' - u' Enter Web Form' - u' or ' - u'Preview Web Form') % { - 'form_id': survey.id_string, - 'form_url': enketo_webform_url - }, - 'form_o': survey + "type": "alert-success", + "preview_url": reverse( + enketo_preview, + kwargs={"username": username, "id_string": survey.id_string}, + ), + "text": _( + f"Successfully published {survey.id_string}." + f' Enter Web Form' + ' or ' + "Preview Web Form" + ), + "form_o": survey, } + form_result = publish_form(set_form) - if form_result['type'] == 'alert-success': + if form_result["type"] == "alert-success": # comment the following condition (and else) # when we want to enable sms check for all. # until then, it checks if form barely related to sms - if is_sms_related(form_result.get('form_o')): + if is_sms_related(form_result.get("form_o")): form_result_sms = check_form_sms_compatibility(form_result) - data['message_list'] = [form_result, form_result_sms] + data["message_list"] = [form_result, form_result_sms] else: - data['message'] = form_result + data["message"] = form_result else: - data['message'] = form_result + data["message"] = form_result # profile view... # for the same user -> dashboard @@ -221,47 +258,68 @@ def set_form(): form = QuickConverterFile() form_url = QuickConverterURL() - request_url = request.build_absolute_uri( - "/%s" % request.user.username) - url = request_url.replace('http://', 'https://') - xforms = XForm.objects.filter(user=content_user, - deleted_at__isnull=True)\ - .select_related('user').only( - 'id', 'id_string', 'downloadable', 'shared', 'shared_data', - 'user__username', 'num_of_submissions', 'title', - 'last_submission_time', 'instances_with_geopoints', - 'encrypted', 'date_created') + request_url = request.build_absolute_uri(f"/{request.user.username}") + url = request_url.replace("http://", "https://") + xforms = ( + XForm.objects.filter(user=content_user, deleted_at__isnull=True) + .select_related("user") + .only( + "id", + "id_string", + "downloadable", + "shared", + "shared_data", + "user__username", + "num_of_submissions", + "title", + "last_submission_time", + "instances_with_geopoints", + "encrypted", + "date_created", + ) + ) user_xforms = xforms # forms shared with user forms_shared_with = get_forms_shared_with_user(content_user).only( - 'id', 'id_string', 'downloadable', 'shared', 'shared_data', - 'user__username', 'num_of_submissions', 'title', - 'last_submission_time', 'instances_with_geopoints', 'encrypted', - 'date_created') + "id", + "id_string", + "downloadable", + "shared", + "shared_data", + "user__username", + "num_of_submissions", + "title", + "last_submission_time", + "instances_with_geopoints", + "encrypted", + "date_created", + ) xforms_list = [ { - 'id': 'published', - 'xforms': user_xforms, - 'title': _(u"Published Forms"), - 'small': _("Export, map, and view submissions.") + "id": "published", + "xforms": user_xforms, + "title": _("Published Forms"), + "small": _("Export, map, and view submissions."), }, { - 'id': 'shared', - 'xforms': forms_shared_with, - 'title': _(u"Shared Forms"), - 'small': _("List of forms shared with you.") - } + "id": "shared", + "xforms": forms_shared_with, + "title": _("Shared Forms"), + "small": _("List of forms shared with you."), + }, ] - data.update({ - 'all_forms': all_forms, - 'show_dashboard': show_dashboard, - 'form': form, - 'form_url': form_url, - 'url': url, - 'user_xforms': user_xforms, - 'xforms_list': xforms_list, - 'forms_shared_with': forms_shared_with - }) + data.update( + { + "all_forms": all_forms, + "show_dashboard": show_dashboard, + "form": form, + "form_url": form_url, + "url": url, + "user_xforms": user_xforms, + "xforms_list": xforms_list, + "forms_shared_with": forms_shared_with, + } + ) # for any other user -> profile set_profile_data(data, content_user) @@ -274,67 +332,87 @@ def set_form(): def members_list(request): + """Show members list page view.""" if not request.user.is_staff and not request.user.is_superuser: - return HttpResponseForbidden(_(u'Forbidden.')) + return HttpResponseForbidden(_("Forbidden.")) users = User.objects.all() - template = 'people.html' + template = "people.html" - return render(request, template, {'template': template, 'users': users}) + return render(request, template, {"template": template, "users": users}) @login_required def profile_settings(request, username): + """User profile settings page view.""" if request.user.username != username: return HttpResponseNotFound("Page not found") content_user = check_and_set_user(request, username) - profile, created = UserProfile.objects.get_or_create(user=content_user) - if request.method == 'POST': - form = UserProfileForm(request.POST, instance=profile) + if isinstance(content_user, str): + return HttpResponseRedirect(content_user) + + user_profile, _created = UserProfile.objects.get_or_create(user=content_user) + if request.method == "POST": + form = UserProfileForm(request.POST, instance=user_profile) if form.is_valid(): # get user # user.email = cleaned_email - form.instance.user.email = form.cleaned_data['email'] + form.instance.user.email = form.cleaned_data["email"] form.instance.user.save() form.save() # todo: add string rep. of settings to see what changed audit = {} audit_log( - Actions.PROFILE_SETTINGS_UPDATED, request.user, content_user, - _("Profile settings updated."), audit, request) - return HttpResponseRedirect(reverse( - public_profile, kwargs={'username': request.user.username} - )) + Actions.PROFILE_SETTINGS_UPDATED, + request.user, + content_user, + _("Profile settings updated."), + audit, + request, + ) + return HttpResponseRedirect( + reverse(public_profile, kwargs={"username": request.user.username}) + ) else: form = UserProfileForm( - instance=profile, initial={"email": content_user.email}) + instance=user_profile, initial={"email": content_user.email} + ) - return render(request, "settings.html", - {'content_user': content_user, 'form': form}) + return render( + request, "settings.html", {"content_user": content_user, "form": form} + ) @require_GET def public_profile(request, username): + """Show user's public profile page view.""" content_user = check_and_set_user(request, username) - if isinstance(content_user, HttpResponseRedirect): - return content_user + if isinstance(content_user, str): + return HttpResponseRedirect(content_user) + data = {} set_profile_data(data, content_user) - data['is_owner'] = request.user == content_user + data["is_owner"] = request.user == content_user audit = {} audit_log( - Actions.PUBLIC_PROFILE_ACCESSED, request.user, content_user, - _("Public profile accessed."), audit, request) + Actions.PUBLIC_PROFILE_ACCESSED, + request.user, + content_user, + _("Public profile accessed."), + audit, + request, + ) return render(request, "profile.html", data) @login_required def dashboard(request): + """Show the dashboard page view.""" content_user = request.user data = { - 'form': QuickConverter(), - 'content_user': content_user, - 'url': request.build_absolute_uri("/%s" % request.user.username) + "form": QuickConverter(), + "content_user": content_user, + "url": request.build_absolute_uri(f"/{request.user.username}"), } set_profile_data(data, content_user) @@ -342,95 +420,107 @@ def dashboard(request): def redirect_to_public_link(request, uuid): + """Redirects to the public link of the form.""" xform = get_object_or_404(XForm, uuid=uuid, deleted_at__isnull=True) - request.session['public_link'] = \ + request.session["public_link"] = ( xform.uuid if MetaData.public_link(xform) else False + ) - return HttpResponseRedirect(reverse(show, kwargs={ - 'username': xform.user.username, - 'id_string': xform.id_string - })) + return HttpResponseRedirect( + reverse( + show, kwargs={"username": xform.user.username, "id_string": xform.id_string} + ) + ) def set_xform_owner_data(data, xform, request, username, id_string): - data['sms_support_form'] = ActivateSMSSupportForm( - initial={'enable_sms_support': xform.allows_sms, - 'sms_id_string': xform.sms_id_string}) + """Set xform owner page view.""" + data["sms_support_form"] = ActivateSMSSupportForm( + initial={ + "enable_sms_support": xform.allows_sms, + "sms_id_string": xform.sms_id_string, + } + ) if not xform.allows_sms: - data['sms_compatible'] = check_form_sms_compatibility( - None, json_survey=json.loads(xform.json)) + data["sms_compatible"] = check_form_sms_compatibility( + None, json_survey=xform.json_dict() + ) else: - url_root = request.build_absolute_uri('/')[:-1] - data['sms_providers_doc'] = providers_doc( - url_root=url_root, - username=username, - id_string=id_string) - data['url_root'] = url_root - - data['form_license_form'] = FormLicenseForm( - initial={'value': data['form_license']}) - data['data_license_form'] = DataLicenseForm( - initial={'value': data['data_license']}) - data['doc_form'] = SupportDocForm() - data['source_form'] = SourceForm() - data['media_form'] = MediaForm() - data['mapbox_layer_form'] = MapboxLayerForm() - data['external_export_form'] = ExternalExportForm() + url_root = request.build_absolute_uri("/")[:-1] + data["sms_providers_doc"] = providers_doc( + url_root=url_root, username=username, id_string=id_string + ) + data["url_root"] = url_root + + data["form_license_form"] = FormLicenseForm(initial={"value": data["form_license"]}) + data["data_license_form"] = DataLicenseForm(initial={"value": data["data_license"]}) + data["doc_form"] = SupportDocForm() + data["source_form"] = SourceForm() + data["media_form"] = MediaForm() + data["mapbox_layer_form"] = MapboxLayerForm() + data["external_export_form"] = ExternalExportForm() users_with_perms = [] for perm in get_xform_users_with_perms(xform).items(): has_perm = [] - if 'change_xform' in perm[1]: - has_perm.append(_(u"Can Edit")) - if 'view_xform' in perm[1]: - has_perm.append(_(u"Can View")) - if 'report_xform' in perm[1]: - has_perm.append(_(u"Can submit to")) - users_with_perms.append((perm[0], u" | ".join(has_perm))) - data['users_with_perms'] = users_with_perms - data['permission_form'] = PermissionForm(username) + if "change_xform" in perm[1]: + has_perm.append(_("Can Edit")) + if "view_xform" in perm[1]: + has_perm.append(_("Can View")) + if "report_xform" in perm[1]: + has_perm.append(_("Can submit to")) + users_with_perms.append((perm[0], " | ".join(has_perm))) + data["users_with_perms"] = users_with_perms + data["permission_form"] = PermissionForm(username) @require_GET def show(request, username=None, id_string=None, uuid=None): + """Show form page view.""" if uuid: return redirect_to_public_link(request, uuid) - xform, is_owner, can_edit, can_view = get_xform_and_perms( - username, id_string, request) + xform, is_xform_owner, can_edit, can_view = get_xform_and_perms( + username, id_string, request + ) # no access - if not (xform.shared or can_view or request.session.get('public_link')): + if not (xform.shared or can_view or request.session.get("public_link")): return HttpResponseRedirect(reverse(home)) data = {} - data['cloned'] = len( - XForm.objects.filter(user__username__iexact=request.user.username, - id_string__iexact=id_string + XForm.CLONED_SUFFIX, - deleted_at__isnull=True) - ) > 0 + data["cloned"] = ( + len( + XForm.objects.filter( + user__username__iexact=request.user.username, + id_string__iexact=id_string + XForm.CLONED_SUFFIX, + deleted_at__isnull=True, + ) + ) + > 0 + ) try: - data['public_link'] = MetaData.public_link(xform) - data['is_owner'] = is_owner - data['can_edit'] = can_edit - data['can_view'] = can_view or request.session.get('public_link') - data['xform'] = xform - data['content_user'] = xform.user - data['base_url'] = "https://%s" % request.get_host() - data['source'] = MetaData.source(xform) - data['form_license'] = MetaData.form_license(xform) - data['data_license'] = MetaData.data_license(xform) - data['supporting_docs'] = MetaData.supporting_docs(xform) - data['media_upload'] = MetaData.media_upload(xform) - data['mapbox_layer'] = MetaData.mapbox_layer_upload(xform) - data['external_export'] = MetaData.external_export(xform) + data["public_link"] = MetaData.public_link(xform) + data["is_owner"] = is_xform_owner + data["can_edit"] = can_edit + data["can_view"] = can_view or request.session.get("public_link") + data["xform"] = xform + data["content_user"] = xform.user + data["base_url"] = f"https://{request.get_host()}" + data["source"] = MetaData.source(xform) + data["form_license"] = MetaData.form_license(xform) + data["data_license"] = MetaData.data_license(xform) + data["supporting_docs"] = MetaData.supporting_docs(xform) + data["media_upload"] = MetaData.media_upload(xform) + data["mapbox_layer"] = MetaData.mapbox_layer_upload(xform) + data["external_export"] = MetaData.external_export(xform) except XLSFormError as e: return HttpResponseBadRequest(e.__str__()) - if is_owner: + if is_xform_owner: set_xform_owner_data(data, xform, request, username, id_string) if xform.allows_sms: - data['sms_support_doc'] = get_autodoc_for(xform) + data["sms_support_doc"] = get_autodoc_for(xform) return render(request, "show.html", data) @@ -438,18 +528,20 @@ def show(request, username=None, id_string=None, uuid=None): @login_required @require_GET def api_token(request, username=None): + """Show user's API Token page view.""" if request.user.username == username: user = get_object_or_404(User, username=username) data = {} - data['token_key'], created = Token.objects.get_or_create(user=user) + data["token_key"], _created = Token.objects.get_or_create(user=user) return render(request, "api_token.html", data) - return HttpResponseForbidden(_(u'Permission denied.')) + return HttpResponseForbidden(_("Permission denied.")) +# pylint: disable=too-many-locals,too-many-branches @require_http_methods(["GET", "OPTIONS"]) -def api(request, username=None, id_string=None): +def api(request, username=None, id_string=None): # noqa C901 """ Returns all results as JSON. If a parameter string is passed, it takes the 'query' parameter, converts this string to a dictionary, an @@ -471,25 +563,25 @@ def api(request, username=None, id_string=None): helper_auth_helper(request) helper_auth_helper(request) - xform, owner = check_and_set_user_and_form(username, id_string, request) + xform, _owner = check_and_set_user_and_form(username, id_string, request) if not xform: - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) - query = request.GET.get('query') + query = request.GET.get("query") total_records = xform.num_of_submissions try: args = { - 'xform': xform, - 'query': query, - 'fields': request.GET.get('fields'), - 'sort': request.GET.get('sort') + "xform": xform, + "query": query, + "fields": request.GET.get("fields"), + "sort": request.GET.get("sort"), } - if 'page' in request.GET: - page = int(request.GET.get('page')) - page_size = request.GET.get('page_size', request.GET.get('limit')) + if "page" in request.GET: + page = int(request.GET.get("page")) + page_size = request.GET.get("page_size", request.GET.get("limit")) if page_size: page_size = int(page_size) @@ -503,36 +595,32 @@ def api(request, username=None, id_string=None): if query: count_args = args.copy() - count_args['count'] = True - count_results = [i for i in query_data(**count_args)] - - if len(count_results): - total_records = count_results[0].get('count', total_records) + count_args["count"] = True + count_results = list(query_data(**count_args)) + if count_results: + total_records = count_results[0].get("count", total_records) - if 'start' in request.GET: - args["start_index"] = int(request.GET.get('start')) + if "start" in request.GET: + args["start_index"] = int(request.GET.get("start")) - if 'limit' in request.GET: - args["limit"] = int(request.GET.get('limit')) + if "limit" in request.GET: + args["limit"] = int(request.GET.get("limit")) - if 'count' in request.GET: - args["count"] = True if int(request.GET.get('count')) > 0\ - else False + if "count" in request.GET: + args["count"] = int(request.GET.get("count")) > 0 cursor = query_data(**args) except (ValueError, TypeError) as e: return HttpResponseBadRequest(conditional_escape(e.__str__())) - try: - response_text = json_util.dumps([i for i in cursor]) - except OperationalError: - return HttpResponseServerError() - - if 'callback' in request.GET and request.GET.get('callback') != '': - callback = request.GET.get('callback') - response_text = ("%s(%s)" % (callback, response_text)) + if "callback" in request.GET and request.GET.get("callback") != "": + callback = request.GET.get("callback") + response_text = json_util.dumps(list(cursor)) + response_text = f"{callback}({response_text})" + response = HttpResponse(response_text) + else: + response = JsonResponse(list(cursor), safe=False) - response = HttpResponse(response_text, content_type='application/json') add_cors_headers(response) return response @@ -543,343 +631,355 @@ def public_api(request, username, id_string): """ Returns public information about the form as JSON """ - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) - _DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S' + datetime_format = "%Y-%m-%d %H:%M:%S" exports = { - 'username': xform.user.username, - 'id_string': xform.id_string, - 'bamboo_dataset': xform.bamboo_dataset, - 'shared': xform.shared, - 'shared_data': xform.shared_data, - 'downloadable': xform.downloadable, - 'title': xform.title, - 'date_created': xform.date_created.strftime(_DATETIME_FORMAT), - 'date_modified': xform.date_modified.strftime(_DATETIME_FORMAT), - 'uuid': xform.uuid, + "username": xform.user.username, + "id_string": xform.id_string, + "bamboo_dataset": xform.bamboo_dataset, + "shared": xform.shared, + "shared_data": xform.shared_data, + "downloadable": xform.downloadable, + "title": xform.title, + "date_created": xform.date_created.strftime(datetime_format), + "date_modified": xform.date_modified.strftime(datetime_format), + "uuid": xform.uuid, } - response_text = json.dumps(exports) - return HttpResponse(response_text, content_type='application/json') + return JsonResponse(exports) +# pylint: disable=too-many-locals,too-many-branches,too-many-statements @login_required -def edit(request, username, id_string): - xform = XForm.objects.get(user__username__iexact=username, - id_string__iexact=id_string, - deleted_at__isnull=True) +def edit(request, username, id_string): # noqa C901 + """Edit form page view.""" + xform = XForm.objects.get( + user__username__iexact=username, + id_string__iexact=id_string, + deleted_at__isnull=True, + ) owner = xform.user - if username == request.user.username or\ - request.user.has_perm('logger.change_xform', xform): - if request.POST.get('description') or\ - request.POST.get('description') == '': - audit = { - 'xform': xform.id_string - } + if username == request.user.username or request.user.has_perm( + "logger.change_xform", xform + ): + if request.POST.get("description") or request.POST.get("description") == "": + audit = {"xform": xform.id_string} + old_description = xform.description + new_description = request.POST["description"] audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Description for '%(id_string)s' updated from " - "'%(old_description)s' to '%(new_description)s'.") % - { - 'id_string': xform.id_string, - 'old_description': xform.description, - 'new_description': request.POST['description'] - }, audit, request) - xform.description = request.POST['description'] - elif request.POST.get('title'): - audit = { - 'xform': xform.id_string - } + Actions.FORM_UPDATED, + request.user, + owner, + _( + f"Description for '{id_string}' updated from " + f"'{old_description}' to '{new_description}'." + ), + audit, + request, + ) + xform.description = request.POST["description"] + elif request.POST.get("title"): + audit = {"xform": xform.id_string} + old_title = (xform.title,) + new_title = (request.POST.get("title"),) audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Title for '%(id_string)s' updated from " - "'%(old_title)s' to '%(new_title)s'.") % - { - 'id_string': xform.id_string, - 'old_title': xform.title, - 'new_title': request.POST.get('title') - }, audit, request) - xform.title = request.POST['title'] - elif request.POST.get('toggle_shared'): - if request.POST['toggle_shared'] == 'data': - audit = { - 'xform': xform.id_string - } + Actions.FORM_UPDATED, + request.user, + owner, + _( + f"Title for '{id_string}' updated from " + f"'{old_title}' to '{new_title}'." + ) + % { + "id_string": xform.id_string, + }, + audit, + request, + ) + xform.title = request.POST["title"] + elif request.POST.get("toggle_shared"): + if request.POST["toggle_shared"] == "data": + audit = {"xform": xform.id_string} + old_shared = _("shared") if xform.shared_data else _("not shared") + new_shared = _("shared") if not xform.shared_data else _("not shared") audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Data sharing updated for '%(id_string)s' from " - "'%(old_shared)s' to '%(new_shared)s'.") % - { - 'id_string': xform.id_string, - 'old_shared': - _("shared") if xform.shared_data else _("not shared"), - 'new_shared': - _("shared") - if not xform.shared_data else _("not shared") - }, audit, request) + Actions.FORM_UPDATED, + request.user, + owner, + _( + f"Data sharing updated for '{id_string}' from " + f"'{old_shared}' to '{new_shared}'." + ), + audit, + request, + ) xform.shared_data = not xform.shared_data - elif request.POST['toggle_shared'] == 'form': - audit = { - 'xform': xform.id_string - } + elif request.POST["toggle_shared"] == "form": + audit = {"xform": xform.id_string} + old_shared = _("shared") if xform.shared else _("not shared") + new_shared = _("shared") if not xform.shared else _("not shared") audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Form sharing for '%(id_string)s' updated " - "from '%(old_shared)s' to '%(new_shared)s'.") % - { - 'id_string': xform.id_string, - 'old_shared': - _("shared") if xform.shared else _("not shared"), - 'new_shared': - _("shared") if not xform.shared else _("not shared") - }, audit, request) + Actions.FORM_UPDATED, + request.user, + owner, + _( + f"Form sharing for '{xform.id_string}' updated " + f"from '{old_shared}' to '{new_shared}'." + ), + audit, + request, + ) xform.shared = not xform.shared - elif request.POST['toggle_shared'] == 'active': - audit = { - 'xform': xform.id_string - } + elif request.POST["toggle_shared"] == "active": + audit = {"xform": xform.id_string} + old_shared = _("shared") if xform.downloadable else _("not shared") + new_shared = _("shared") if not xform.downloadable else _("not shared") audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Active status for '%(id_string)s' updated from " - "'%(old_shared)s' to '%(new_shared)s'.") % - { - 'id_string': xform.id_string, - 'old_shared': - _("shared") if xform.downloadable else _("not shared"), - 'new_shared': - _("shared") - if not xform.downloadable else _("not shared") - }, audit, request) + Actions.FORM_UPDATED, + request.user, + owner, + _( + f"Active status for '{xform.id_string}' updated from " + f"'{old_shared}' to '{new_shared}'." + ), + audit, + request, + ) xform.downloadable = not xform.downloadable - elif request.POST.get('form-license'): - audit = { - 'xform': xform.id_string - } + elif request.POST.get("form-license"): + audit = {"xform": xform.id_string} + form_license = request.POST["form-license"] audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Form License for '%(id_string)s' updated to " - "'%(form_license)s'.") % - { - 'id_string': xform.id_string, - 'form_license': request.POST['form-license'], - }, audit, request) - MetaData.form_license(xform, request.POST['form-license']) - elif request.POST.get('data-license'): - audit = { - 'xform': xform.id_string - } + Actions.FORM_UPDATED, + request.user, + owner, + _( + f"Form License for '{xform.id_string}' updated to " + f"'{form_license}'." + ), + audit, + request, + ) + MetaData.form_license(xform, request.POST["form-license"]) + elif request.POST.get("data-license"): + audit = {"xform": xform.id_string} + data_license = request.POST["data-license"] audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Data license for '%(id_string)s' updated to " - "'%(data_license)s'.") % - { - 'id_string': xform.id_string, - 'data_license': request.POST['data-license'], - }, audit, request) - MetaData.data_license(xform, request.POST['data-license']) - elif request.POST.get('source') or request.FILES.get('source'): - audit = { - 'xform': xform.id_string - } + Actions.FORM_UPDATED, + request.user, + owner, + _( + f"Data license for '{xform.id_string}' updated to " + f"'{data_license}'." + ), + audit, + request, + ) + MetaData.data_license(xform, request.POST["data-license"]) + elif request.POST.get("source") or request.FILES.get("source"): + audit = {"xform": xform.id_string} + source = request.POST.get("source") audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Source for '%(id_string)s' updated to '%(source)s'.") % - { - 'id_string': xform.id_string, - 'source': request.POST.get('source'), - }, audit, request) - MetaData.source(xform, request.POST.get('source'), - request.FILES.get('source')) - elif request.POST.get('enable_sms_support_trigger') is not None: + Actions.FORM_UPDATED, + request.user, + owner, + _(f"Source for '{xform.id_string}' updated to '{source}'."), + audit, + request, + ) + MetaData.source( + xform, request.POST.get("source"), request.FILES.get("source") + ) + elif request.POST.get("enable_sms_support_trigger") is not None: sms_support_form = ActivateSMSSupportForm(request.POST) if sms_support_form.is_valid(): - audit = { - 'xform': xform.id_string - } - enabled = \ - sms_support_form.cleaned_data.get('enable_sms_support') + audit = {"xform": xform.id_string} + enabled = sms_support_form.cleaned_data.get("enable_sms_support") if enabled: audit_action = Actions.SMS_SUPPORT_ACTIVATED - audit_message = _(u"SMS Support Activated on") + audit_message = _("SMS Support Activated on") else: audit_action = Actions.SMS_SUPPORT_DEACTIVATED - audit_message = _(u"SMS Support Deactivated on") + audit_message = _("SMS Support Deactivated on") audit_log( - audit_action, request.user, owner, - audit_message - % {'id_string': xform.id_string}, audit, request) + audit_action, + request.user, + owner, + audit_message, + audit, + request, + ) # stored previous states to be able to rollback form status # in case we can't save. - pe = xform.allows_sms + previous_allow_sms = xform.allows_sms pid = xform.sms_id_string xform.allows_sms = enabled - xform.sms_id_string = \ - sms_support_form.cleaned_data.get('sms_id_string') - compat = check_form_sms_compatibility(None, - json.loads(xform.json)) - if compat['type'] == 'alert-error': + xform.sms_id_string = sms_support_form.cleaned_data.get("sms_id_string") + compat = check_form_sms_compatibility(None, xform.json_dict()) + if compat["type"] == "alert-error": xform.allows_sms = False xform.sms_id_string = pid try: xform.save() except IntegrityError: # unfortunately, there's no feedback mechanism here - xform.allows_sms = pe + xform.allows_sms = previous_allow_sms xform.sms_id_string = pid - elif request.POST.get('media_url'): - uri = request.POST.get('media_url') + elif request.POST.get("media_url"): + uri = request.POST.get("media_url") MetaData.media_add_uri(xform, uri) - elif request.FILES.get('media'): - audit = { - 'xform': xform.id_string - } + elif request.FILES.get("media"): + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Media added to '%(id_string)s'.") % - { - 'id_string': xform.id_string - }, audit, request) - for aFile in request.FILES.getlist("media"): - MetaData.media_upload(xform, aFile) - elif request.POST.get('map_name'): + Actions.FORM_UPDATED, + request.user, + owner, + _(f"Media added to '{xform.id_string}'."), + audit, + request, + ) + for media_file in request.FILES.getlist("media"): + MetaData.media_upload(xform, media_file) + elif request.POST.get("map_name"): mapbox_layer = MapboxLayerForm(request.POST) if mapbox_layer.is_valid(): - audit = { - 'xform': xform.id_string - } + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Map layer added to '%(id_string)s'.") % - { - 'id_string': xform.id_string - }, audit, request) + Actions.FORM_UPDATED, + request.user, + owner, + _(f"Map layer added to '{xform.id_string}'."), + audit, + request, + ) MetaData.mapbox_layer_upload(xform, mapbox_layer.cleaned_data) - elif request.FILES.get('doc'): - audit = { - 'xform': xform.id_string - } + elif request.FILES.get("doc"): + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Supporting document added to '%(id_string)s'.") % - { - 'id_string': xform.id_string - }, audit, request) - MetaData.supporting_docs(xform, request.FILES.get('doc')) - elif request.POST.get("template_token") \ - and request.POST.get("template_token"): + Actions.FORM_UPDATED, + request.user, + owner, + _(f"Supporting document added to '{xform.id_string}'."), + audit, + request, + ) + MetaData.supporting_docs(xform, request.FILES.get("doc")) + elif request.POST.get("template_token") and request.POST.get("template_token"): template_name = request.POST.get("template_name") template_token = request.POST.get("template_token") - audit = { - 'xform': xform.id_string - } + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("External export added to '%(id_string)s'.") % - { - 'id_string': xform.id_string - }, audit, request) - merged = template_name + '|' + template_token + Actions.FORM_UPDATED, + request.user, + owner, + _(f"External export added to '{xform.id_string}'."), + audit, + request, + ) + merged = template_name + "|" + template_token MetaData.external_export(xform, merged) - elif request.POST.get("external_url") \ - and request.FILES.get("xls_template"): + elif request.POST.get("external_url") and request.FILES.get("xls_template"): template_upload_name = request.POST.get("template_upload_name") external_url = request.POST.get("external_url") xls_template = request.FILES.get("xls_template") - result = upload_template_for_external_export(external_url, - xls_template) - status_code = result.split('|')[0] - token = result.split('|')[1] - if status_code == '201': - data_value =\ - template_upload_name + '|' + external_url + '/xls/' + token + result = upload_template_for_external_export(external_url, xls_template) + status_code = result.split("|")[0] + token = result.split("|")[1] + if status_code == "201": + data_value = template_upload_name + "|" + external_url + "/xls/" + token MetaData.external_export(xform, data_value=data_value) xform.update() if request.is_ajax(): - return HttpResponse(_(u'Updated succeeded.')) - else: - return HttpResponseRedirect(reverse(show, kwargs={ - 'username': username, - 'id_string': id_string - })) + return HttpResponse(_("Updated succeeded.")) + return HttpResponseRedirect( + reverse(show, kwargs={"username": username, "id_string": id_string}) + ) - return HttpResponseForbidden(_(u'Update failed.')) + return HttpResponseForbidden(_("Update failed.")) def getting_started(request): - template = 'getting_started.html' + """The getting started page view.""" + template = "getting_started.html" - return render(request, 'base.html', {'template': template}) + return render(request, "base.html", {"template": template}) def support(request): - template = 'support.html' + """The support page view.""" + template = "support.html" - return render(request, 'base.html', {'template': template}) + return render(request, "base.html", {"template": template}) def faq(request): - template = 'faq.html' + """The frequently asked questions page view.""" + template = "faq.html" - return render(request, 'base.html', {'template': template}) + return render(request, "base.html", {"template": template}) def xls2xform(request): - template = 'xls2xform.html' + """The XLSForm to XForm page view.""" + template = "xls2xform.html" - return render(request, 'base.html', {'template': template}) + return render(request, "base.html", {"template": template}) def tutorial(request): - template = 'tutorial.html' - username = request.user.username if request.user.username else \ - 'your-user-name' - url = request.build_absolute_uri("/%s" % username) + """The tutorial page view.""" + template = "tutorial.html" + username = request.user.username if request.user.username else "your-user-name" + url = request.build_absolute_uri(f"/{username}") - return render(request, 'base.html', {'template': template, 'url': url}) + return render(request, "base.html", {"template": template, "url": url}) def resources(request): - if 'fr' in request.LANGUAGE_CODE.lower(): - deck_id = 'a351f6b0a3730130c98b12e3c5740641' - else: - deck_id = '1a33a070416b01307b8022000a1de118' + """The resources page view.""" + deck_id = "1a33a070416b01307b8022000a1de118" + if "fr" in request.LANGUAGE_CODE.lower(): + deck_id = "a351f6b0a3730130c98b12e3c5740641" - return render(request, 'resources.html', {'deck_id': deck_id}) + return render(request, "resources.html", {"deck_id": deck_id}) def about_us(request): - a_flatpage = '/about-us/' - username = request.user.username if request.user.username else \ - 'your-user-name' - url = request.build_absolute_uri("/%s" % username) + """The about us page view""" + a_flatpage = "/about-us/" + username = request.user.username if request.user.username else "your-user-name" + url = request.build_absolute_uri(f"/{username}") - return render(request, 'base.html', {'a_flatpage': a_flatpage, 'url': url}) + return render(request, "base.html", {"a_flatpage": a_flatpage, "url": url}) def privacy(request): - template = 'privacy.html' + """The privacy page view.""" + template = "privacy.html" - return render(request, 'base.html', {'template': template}) + return render(request, "base.html", {"template": template}) def tos(request): - template = 'tos.html' + """The terms of service page view.""" + template = "tos.html" - return render(request, 'base.html', {'template': template}) + return render(request, "base.html", {"template": template}) def syntax(request): - template = 'syntax.html' + """The XLSForm Syntax page view.""" + template = "syntax.html" - return render(request, 'base.html', {'template': template}) + return render(request, "base.html", {"template": template}) def form_gallery(request): @@ -889,535 +989,536 @@ def form_gallery(request): """ data = {} if request.user.is_authenticated: - data['loggedin_user'] = request.user - data['shared_forms'] = XForm.objects.filter(shared=True, - deleted_at__isnull=True) + data["loggedin_user"] = request.user + data["shared_forms"] = XForm.objects.filter(shared=True, deleted_at__isnull=True) # build list of shared forms with cloned suffix id_strings_with_cloned_suffix = [ - x.id_string + XForm.CLONED_SUFFIX for x in data['shared_forms'] + x.id_string + XForm.CLONED_SUFFIX for x in data["shared_forms"] ] # build list of id_strings for forms this user has cloned - data['cloned'] = [ + data["cloned"] = [ x.id_string.split(XForm.CLONED_SUFFIX)[0] for x in XForm.objects.filter( user__username__iexact=request.user.username, id_string__in=id_strings_with_cloned_suffix, - deleted_at__isnull=True + deleted_at__isnull=True, ) ] - return render(request, 'form_gallery.html', data) + return render(request, "form_gallery.html", data) def download_metadata(request, username, id_string, data_id): - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + """Downloads metadata file contents.""" + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) owner = xform.user if username == request.user.username or xform.shared: data = get_object_or_404(MetaData, pk=data_id) file_path = data.data_file.name - filename, extension = os.path.splitext(file_path.split('/')[-1]) - extension = extension.strip('.') + filename, extension = os.path.splitext(file_path.split("/")[-1]) + extension = extension.strip(".") dfs = get_storage_class()() if dfs.exists(file_path): - audit = { - 'xform': xform.id_string - } + audit = {"xform": xform.id_string} + filename_w_extension = f"{filename}.{extension}" audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Document '%(filename)s' for '%(id_string)s' downloaded.") % - { - 'id_string': xform.id_string, - 'filename': "%s.%s" % (filename, extension) - }, audit, request) + Actions.FORM_UPDATED, + request.user, + owner, + _( + f"Document '{filename_w_extension}' for " + f"'{xform.id_string}' downloaded." + ), + audit, + request, + ) response = response_with_mimetype_and_name( data.data_file_type, - filename, extension=extension, show_date=False, - file_path=file_path) + filename, + extension=extension, + show_date=False, + file_path=file_path, + ) return response - else: - return HttpResponseNotFound() + return HttpResponseNotFound() - return HttpResponseForbidden(_(u'Permission denied.')) + return HttpResponseForbidden(_("Permission denied.")) @login_required() def delete_metadata(request, username, id_string, data_id): - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + """Deletes a metadata record.""" + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) owner = xform.user data = get_object_or_404(MetaData, pk=data_id) dfs = get_storage_class()() req_username = request.user.username - if request.GET.get('del', False) and username == req_username: - try: - dfs.delete(data.data_file.name) - data.delete() - audit = { - 'xform': xform.id_string - } - audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Document '%(filename)s' deleted from '%(id_string)s'.") % - { - 'id_string': xform.id_string, - 'filename': os.path.basename(data.data_file.name) - }, audit, request) - return HttpResponseRedirect(reverse(show, kwargs={ - 'username': username, - 'id_string': id_string - })) - except Exception: - return HttpResponseServerError() - elif (request.GET.get('map_name_del', False) or - request.GET.get('external_del', False)) and username == req_username: + if request.GET.get("del", False) and username == req_username: + dfs.delete(data.data_file.name) data.delete() - audit = { - 'xform': xform.id_string - } + audit = {"xform": xform.id_string} + filename = (os.path.basename(data.data_file.name),) audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Map layer deleted from '%(id_string)s'.") % - { - 'id_string': xform.id_string, - }, audit, request) - return HttpResponseRedirect(reverse(show, kwargs={ - 'username': username, - 'id_string': id_string - })) + Actions.FORM_UPDATED, + request.user, + owner, + _(f"Document '{filename}' deleted from '{xform.id_string}'."), + audit, + request, + ) + return HttpResponseRedirect( + reverse(show, kwargs={"username": username, "id_string": id_string}) + ) + if ( + request.GET.get("map_name_del", False) or request.GET.get("external_del", False) + ) and username == req_username: + data.delete() + audit = {"xform": xform.id_string} + audit_log( + Actions.FORM_UPDATED, + request.user, + owner, + _(f"Map layer deleted from '{xform.id_string}'."), + audit, + request, + ) + return HttpResponseRedirect( + reverse(show, kwargs={"username": username, "id_string": id_string}) + ) - return HttpResponseForbidden(_(u'Permission denied.')) + return HttpResponseForbidden(_("Permission denied.")) def download_media_data(request, username, id_string, data_id): + """Redirects to a form metadata record for download.""" xform = get_object_or_404( - XForm, user__username__iexact=username, deleted_at__isnull=True, - id_string__iexact=id_string) + XForm, + user__username__iexact=username, + deleted_at__isnull=True, + id_string__iexact=id_string, + ) owner = xform.user data = get_object_or_404(MetaData, id=data_id) dfs = get_storage_class()() - if request.GET.get('del', False): + if request.GET.get("del", False): if username == request.user.username: - try: - # ensure filename is not an empty string - if data.data_file.name != '': - dfs.delete(data.data_file.name) - - data.delete() - audit = { - 'xform': xform.id_string - } - audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Media download '%(filename)s' deleted from " - "'%(id_string)s'.") % - { - 'id_string': xform.id_string, - 'filename': os.path.basename(data.data_file.name) - }, audit, request) - return HttpResponseRedirect(reverse(show, kwargs={ - 'username': username, - 'id_string': id_string - })) - except Exception as e: - return HttpResponseServerError(e) + # ensure filename is not an empty string + if data.data_file.name != "": + dfs.delete(data.data_file.name) + + data.delete() + audit = {"xform": xform.id_string} + audit_log( + Actions.FORM_UPDATED, + request.user, + owner, + _( + f"Media download '{os.path.basename(data.data_file.name)}'" + f" deleted from '{xform.id_string}'." + ), + audit, + request, + ) + return HttpResponseRedirect( + reverse(show, kwargs={"username": username, "id_string": id_string}) + ) else: if username: # == request.user.username or xform.shared: - if data.data_file.name == '' and data.data_value is not None: + if data.data_file.name == "" and data.data_value is not None: return HttpResponseRedirect(data.data_value) file_path = data.data_file.name - filename, extension = os.path.splitext(file_path.split('/')[-1]) - extension = extension.strip('.') + filename, extension = os.path.splitext(file_path.split("/")[-1]) + extension = extension.strip(".") if dfs.exists(file_path): - audit = { - 'xform': xform.id_string - } + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Media '%(filename)s' downloaded from '%(id_string)s'.") - % { - 'id_string': xform.id_string, - 'filename': os.path.basename(file_path) - }, audit, request) + Actions.FORM_UPDATED, + request.user, + owner, + _( + f"Media '{os.path.basename(file_path)}' " + f"downloaded from '{xform.id_string}'." + ), + audit, + request, + ) response = response_with_mimetype_and_name( data.data_file_type, - filename, extension=extension, show_date=False, - file_path=file_path) + filename, + extension=extension, + show_date=False, + file_path=file_path, + ) return response - else: - return HttpResponseNotFound() + return HttpResponseNotFound() - return HttpResponseForbidden(_(u'Permission denied.')) + return HttpResponseForbidden(_("Permission denied.")) def form_photos(request, username, id_string): + """View form image attachments.""" xform, owner = check_and_set_user_and_form(username, id_string, request) if not xform: - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) data = {} - data['form_view'] = True - data['content_user'] = owner - data['xform'] = xform + data["form_view"] = True + data["content_user"] = owner + data["xform"] = xform image_urls = [] for instance in xform.instances.filter(deleted_at__isnull=True): for attachment in instance.attachments.all(): # skip if not image e.g video or file - if not attachment.mimetype.startswith('image'): + if not attachment.mimetype.startswith("image"): continue data = {} - for i in [u'small', u'medium', u'large', u'original']: - url = reverse(attachment_url, kwargs={'size': i}) - url = '%s?media_file=%s' % (url, attachment.media_file.name) + for i in ["small", "medium", "large", "original"]: + url = reverse(attachment_url, kwargs={"size": i}) + url = f"{url}?media_file={attachment.media_file.name}" data[i] = url image_urls.append(data) image_urls = json.dumps(image_urls) - data['images'] = image_urls - data['profile'], created = UserProfile.objects.get_or_create(user=owner) + data["images"] = image_urls + data["profile"], _created = UserProfile.objects.get_or_create(user=owner) - return render(request, 'form_photos.html', data) + return render(request, "form_photos.html", data) +# pylint: disable=too-many-branches @require_POST -def set_perm(request, username, id_string): - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) +def set_perm(request, username, id_string): # noqa C901 + """Assign form permissions to a user.""" + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) owner = xform.user - if username != request.user.username\ - and not has_permission(xform, username, request): - return HttpResponseForbidden(_(u'Permission denied.')) + if username != request.user.username and not has_permission( + xform, username, request + ): + return HttpResponseForbidden(_("Permission denied.")) try: - perm_type = request.POST['perm_type'] - for_user = request.POST['for_user'] + perm_type = request.POST["perm_type"] + for_user = request.POST["for_user"] except KeyError: return HttpResponseBadRequest() - if perm_type in ['edit', 'view', 'report', 'remove']: + if perm_type in ["edit", "view", "report", "remove"]: try: user = User.objects.get(username=for_user) except User.DoesNotExist: messages.add_message( - request, messages.INFO, - _(u"Wrong username %s." % for_user), - extra_tags='alert-error') + request, + messages.INFO, + _(f"Wrong username {for_user}."), + extra_tags="alert-error", + ) else: - if perm_type == 'edit' and\ - not user.has_perm('change_xform', xform): - audit = { - 'xform': xform.id_string - } + if perm_type == "edit" and not user.has_perm("change_xform", xform): + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_PERMISSIONS_UPDATED, request.user, owner, - _("Edit permissions on '%(id_string)s' assigned to " - "'%(for_user)s'.") % - { - 'id_string': xform.id_string, - 'for_user': for_user - }, audit, request) - assign_perm('change_xform', user, xform) - elif perm_type == 'view' and\ - not user.has_perm('view_xform', xform): - audit = { - 'xform': xform.id_string - } + Actions.FORM_PERMISSIONS_UPDATED, + request.user, + owner, + _( + f"Edit permissions on '{xform.id_string}' assigned to " + f"'{for_user}'." + ), + audit, + request, + ) + assign_perm("change_xform", user, xform) + elif perm_type == "view" and not user.has_perm("view_xform", xform): + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_PERMISSIONS_UPDATED, request.user, owner, - _("View permissions on '%(id_string)s' " - "assigned to '%(for_user)s'.") % - { - 'id_string': xform.id_string, - 'for_user': for_user - }, audit, request) - assign_perm('view_xform', user, xform) - elif perm_type == 'report' and\ - not user.has_perm('report_xform', xform): - audit = { - 'xform': xform.id_string - } + Actions.FORM_PERMISSIONS_UPDATED, + request.user, + owner, + _( + f"View permissions on '{xform.id_string}' " + f"assigned to '{for_user}'." + ) + % {"id_string": xform.id_string, "for_user": for_user}, + audit, + request, + ) + assign_perm("view_xform", user, xform) + elif perm_type == "report" and not user.has_perm("report_xform", xform): + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_PERMISSIONS_UPDATED, request.user, owner, - _("Report permissions on '%(id_string)s' " - "assigned to '%(for_user)s'.") % - { - 'id_string': xform.id_string, - 'for_user': for_user - }, audit, request) - assign_perm('report_xform', user, xform) - elif perm_type == 'remove': - audit = { - 'xform': xform.id_string - } + Actions.FORM_PERMISSIONS_UPDATED, + request.user, + owner, + _( + f"Report permissions on '{xform.id_string}' " + f"assigned to '{for_user}'." + ), + audit, + request, + ) + assign_perm("report_xform", user, xform) + elif perm_type == "remove": + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_PERMISSIONS_UPDATED, request.user, owner, - _("All permissions on '%(id_string)s' " - "removed from '%(for_user)s'.") % - { - 'id_string': xform.id_string, - 'for_user': for_user - }, audit, request) - remove_perm('change_xform', user, xform) - remove_perm('view_xform', user, xform) - remove_perm('report_xform', user, xform) - elif perm_type == 'link': + Actions.FORM_PERMISSIONS_UPDATED, + request.user, + owner, + _( + f"All permissions on '{xform.id_string}' " + f"removed from '{for_user}'." + ), + audit, + request, + ) + remove_perm("change_xform", user, xform) + remove_perm("view_xform", user, xform) + remove_perm("report_xform", user, xform) + elif perm_type == "link": current = MetaData.public_link(xform) - if for_user == 'all': + if for_user == "all": MetaData.public_link(xform, True) - elif for_user == 'none': + elif for_user == "none": MetaData.public_link(xform, False) - elif for_user == 'toggle': + elif for_user == "toggle": MetaData.public_link(xform, not current) - audit = { - 'xform': xform.id_string - } + audit = {"xform": xform.id_string} + action = "removed" + if for_user == "all" or (for_user == "toggle" and not current): + action = "created" audit_log( - Actions.FORM_PERMISSIONS_UPDATED, request.user, owner, - _("Public link on '%(id_string)s' %(action)s.") % - { - 'id_string': xform.id_string, - 'action': - "created" if for_user == "all" or - (for_user == "toggle" and not current) else "removed" - }, audit, request) + Actions.FORM_PERMISSIONS_UPDATED, + request.user, + owner, + _(f"Public link on '{xform.id_string}' {action}."), + audit, + request, + ) if request.is_ajax(): - return HttpResponse( - json.dumps( - {'status': 'success'}), content_type='application/json') + return JsonResponse({"status": "success"}) - return HttpResponseRedirect(reverse(show, kwargs={ - 'username': username, - 'id_string': id_string - })) + return HttpResponseRedirect( + reverse(show, kwargs={"username": username, "id_string": id_string}) + ) @require_POST @login_required def delete_data(request, username=None, id_string=None): + """Delete submission record.""" xform, owner = check_and_set_user_and_form(username, id_string, request) - response_text = u'' + response_text = "" if not xform: - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) - data_id = request.POST.get('id') + data_id = request.POST.get("id") if not data_id: - return HttpResponseBadRequest(_(u"id must be specified")) + return HttpResponseBadRequest(_("id must be specified")) Instance.set_deleted_at(data_id, user=request.user) - audit = { - 'xform': xform.id_string - } + audit = {"xform": xform.id_string} audit_log( - Actions.SUBMISSION_DELETED, request.user, owner, - _("Deleted submission with id '%(record_id)s' on '%(id_string)s'.") % - { - 'id_string': xform.id_string, - 'record_id': data_id - }, audit, request) - response_text = json.dumps({"success": "Deleted data %s" % data_id}) - if 'callback' in request.GET and request.GET.get('callback') != '': - callback = request.GET.get('callback') - response_text = ("%s(%s)" % (callback, response_text)) - - return HttpResponse(response_text, content_type='application/json') + Actions.SUBMISSION_DELETED, + request.user, + owner, + _(f"Deleted submission with id '{data_id}' on '{xform.id_string}'."), + audit, + request, + ) + response_data = {"success": f"Deleted data {data_id}"} + if "callback" in request.GET and request.GET.get("callback") != "": + response_text = json.dumps(response_data) + callback = request.GET.get("callback") + response_text = f"{callback}({response_text})" + return HttpResponse(response_text) + + return JsonResponse(response_data) @require_POST @is_owner def update_xform(request, username, id_string): - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } + """Update a form page view.""" + xform_kwargs = {"id_string__iexact": id_string, "user__username__iexact": username} xform = get_form(xform_kwargs) owner = xform.user def set_form(): + """Publishes the XLSForm""" form = QuickConverter(request.POST, request.FILES) survey = form.publish(request.user, id_string).survey enketo_webform_url = reverse( - enter_data, - kwargs={'username': username, 'id_string': survey.id_string} + enter_data, kwargs={"username": username, "id_string": survey.id_string} ) - audit = { - 'xform': xform.id_string - } + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_XLS_UPDATED, request.user, owner, - _("XLS for '%(id_string)s' updated.") % - { - 'id_string': xform.id_string, - }, audit, request) + Actions.FORM_XLS_UPDATED, + request.user, + owner, + _(f"XLS for '{xform.id_string}' updated."), + audit, + request, + ) return { - 'type': 'alert-success', - 'text': _(u'Successfully published %(form_id)s.' - u' Enter Web Form' - u' or ' - u'Preview Web Form') - % {'form_id': survey.id_string, - 'form_url': enketo_webform_url} + "type": "alert-success", + "text": _( + f"Successfully published {survey.id_string}." + f' Enter Web Form' + ' or ' + "Preview Web Form" + ), } + message = publish_form(set_form) messages.add_message( - request, messages.INFO, message['text'], extra_tags=message['type']) + request, messages.INFO, message["text"], extra_tags=message["type"] + ) - return HttpResponseRedirect(reverse(show, kwargs={ - 'username': username, - 'id_string': id_string - })) + return HttpResponseRedirect( + reverse(show, kwargs={"username": username, "id_string": id_string}) + ) @is_owner def activity(request, username): + """The activity/audit view for the given ``username``.""" owner = get_object_or_404(User, username=username) - return render(request, 'activity.html', {'user': owner}) + return render(request, "activity.html", {"user": owner}) def activity_fields(request): + """Returns Activity/Audit fields in JSON format.""" fields = [ { - 'id': 'created_on', - 'label': _('Performed On'), - 'type': 'datetime', - 'searchable': False - }, - { - 'id': 'action', - 'label': _('Action'), - 'type': 'string', - 'searchable': True, - 'options': sorted([Actions[e] for e in Actions.enums]) - }, - { - 'id': 'user', - 'label': 'Performed By', - 'type': 'string', - 'searchable': True + "id": "created_on", + "label": _("Performed On"), + "type": "datetime", + "searchable": False, }, { - 'id': 'msg', - 'label': 'Description', - 'type': 'string', - 'searchable': True + "id": "action", + "label": _("Action"), + "type": "string", + "searchable": True, + "options": sorted([Actions[e] for e in Actions.enums]), }, + {"id": "user", "label": "Performed By", "type": "string", "searchable": True}, + {"id": "msg", "label": "Description", "type": "string", "searchable": True}, ] - response_text = json.dumps(fields) - return HttpResponse(response_text, content_type='application/json') + return JsonResponse(fields, safe=False) @is_owner def activity_api(request, username): - from bson.objectid import ObjectId + """Returns Audit activity data in JSON format""" def stringify_unknowns(obj): + """Stringify some objects - for use with json.dumps.""" if isinstance(obj, ObjectId): return str(obj) if isinstance(obj, datetime): return obj.strftime(DATETIME_FORMAT) return None + try: - fields = request.GET.get('fields') - query = request.GET.get('query') - sort = request.GET.get('sort') + fields = request.GET.get("fields") + query = request.GET.get("query") + sort = request.GET.get("sort") query_args = { - 'username': username, - 'query': json.loads(query) if query else {}, - 'fields': json.loads(fields) if fields else [], - 'sort': json.loads(sort) if sort else [], + "username": username, + "query": json.loads(query) if query else {}, + "fields": json.loads(fields) if fields else [], + "sort": json.loads(sort) if sort else [], } - if 'start' in request.GET: - query_args["start"] = int(request.GET.get('start')) - if 'limit' in request.GET: - query_args["limit"] = int(request.GET.get('limit')) - if 'count' in request.GET: - query_args["count"] = True \ - if int(request.GET.get('count')) > 0 else False + if "start" in request.GET: + query_args["start"] = int(request.GET.get("start")) + if "limit" in request.GET: + query_args["limit"] = int(request.GET.get("limit")) + if "count" in request.GET: + query_args["count"] = int(request.GET.get("count")) > 0 cursor = AuditLog.query_data(**query_args) except ValueError as e: return HttpResponseBadRequest(e.__str__()) records = list(record for record in cursor) - response_text = json.dumps(records, default=stringify_unknowns) - if 'callback' in request.GET and request.GET.get('callback') != '': - callback = request.GET.get('callback') - response_text = ("%s(%s)" % (callback, response_text)) + if "callback" in request.GET and request.GET.get("callback") != "": + callback = request.GET.get("callback") + response_text = json.dumps(records, default=stringify_unknowns) + response_text = f"{callback}({response_text})" + return HttpResponse(response_text) - return HttpResponse(response_text, content_type='application/json') + return JsonResponse( + records, json_dumps_params={"default": stringify_unknowns}, safe=False + ) def qrcode(request, username, id_string): - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } + """Returns the Enketo URL in QR code image format.""" + xform_kwargs = {"id_string__iexact": id_string, "user__username__iexact": username} xform = get_form(xform_kwargs) try: - formhub_url = "http://%s/" % request.META['HTTP_HOST'] + formhub_url = f"http://{request.META['HTTP_HOST']}/" except KeyError: formhub_url = "http://formhub.org/" - formhub_url = formhub_url + username + '/%s' % xform.pk + formhub_url = formhub_url + username + f"/{xform.pk}" if settings.TESTING_MODE: - formhub_url = "https://{}/{}".format(settings.TEST_HTTP_HOST, - settings.TEST_USERNAME) + formhub_url = f"https://{settings.TEST_HTTP_HOST}/{settings.TEST_USERNAME}" - results = _(u"Unexpected Error occured: No QRCODE generated") - status = 200 + results = _("Unexpected Error occured: No QRCODE generated") + status = HTTPStatus.OK try: enketo_urls = get_enketo_urls(formhub_url, id_string) - except Exception as e: - error_msg = _(u"Error Generating QRCODE: %s" % e) - results = """
%s
""" % error_msg - status = 400 + except EnketoError as e: + error_msg = _(f"Error Generating QRCODE: {e}") + results = f"""
{error_msg}
""" + status = HTTPStatus.BAD_REQUEST else: if enketo_urls: url = enketo_urls.get("url") image = generate_qrcode(url) - results = """%s -
%s""" \ - % (image, url, url, url) + results = f"""{url} +
{url}""" + else: - status = 400 + status = HTTPStatus.BAD_REQUEST - return HttpResponse(results, content_type='text/html', status=status) + return HttpResponse(results, content_type="text/html", status=status) def enketo_preview(request, username, id_string): - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + """Redirects a user to the Enketo preview URL for the given form ``id_string``.""" + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) owner = xform.user if not has_permission(xform, owner, request, xform.shared): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) try: - enketo_urls = get_enketo_urls( - xform.url, xform.id_string) + enketo_urls = get_enketo_urls(xform.url, xform.id_string) - enketo_preview_url = enketo_urls.get('preview_url') + enketo_preview_url = enketo_urls.get("preview_url") except EnketoError as e: return HttpResponse(e) @@ -1434,43 +1535,48 @@ def service_health(request): service_statuses = {} # Check if Database connections are present & data is retrievable - for database in getattr(settings, 'DATABASES').keys(): + for database in getattr(settings, "DATABASES").keys(): + # pylint: disable=broad-except try: XForm.objects.using(database).first() except Exception as e: - service_statuses[f'{database}-Database'] = f'Degraded state; {e}' + service_statuses[f"{database}-Database"] = f"Degraded state; {e}" service_degraded = True else: - service_statuses[f'{database}-Database'] = 'OK' + service_statuses[f"{database}-Database"] = "OK" # Check if cache is accessible + # pylint: disable=broad-except try: - cache.set('ping', 'pong') - cache.delete('ping') + cache.set("ping", "pong") + cache.delete("ping") except Exception as e: - service_statuses['Cache-Service'] = f'Degraded state; {e}' + service_statuses["Cache-Service"] = f"Degraded state; {e}" else: - service_statuses['Cache-Service'] = 'OK' + service_statuses["Cache-Service"] = "OK" - return HttpResponse( - json.dumps(service_statuses), - status=500 if service_degraded else 200, - content_type='application/json') + return JsonResponse( + service_statuses, + status=HTTPStatus.INTERNAL_SERVER_ERROR if service_degraded else HTTPStatus.OK, + ) @require_GET @login_required def username_list(request): + """Show's the list of usernames.""" data = [] - query = request.GET.get('query', None) + query = request.GET.get("query", None) if query: - users = User.objects.values('username')\ - .filter(username__startswith=query, is_active=True, pk__gte=0) - data = [user['username'] for user in users] + users = User.objects.values("username").filter( + username__startswith=query, is_active=True, pk__gte=0 + ) + data = [user["username"] for user in users] - return HttpResponse(json.dumps(data), content_type='application/json') + return JsonResponse(data, safe=False) +# pylint: disable=too-few-public-methods class OnaAuthorizationView(AuthorizationView): """ @@ -1479,7 +1585,8 @@ class OnaAuthorizationView(AuthorizationView): """ def get_context_data(self, **kwargs): - context = super(OnaAuthorizationView, self).get_context_data(**kwargs) - context['user'] = self.request.user - context['request_path'] = self.request.get_full_path() + """Adds `user` and `request_path` to context and returns the context.""" + context = super().get_context_data(**kwargs) + context["user"] = self.request.user + context["request_path"] = self.request.get_full_path() return context diff --git a/onadata/apps/messaging/constants.py b/onadata/apps/messaging/constants.py index be5e1f04e2..dfe7599ddf 100644 --- a/onadata/apps/messaging/constants.py +++ b/onadata/apps/messaging/constants.py @@ -7,7 +7,7 @@ from builtins import str as text -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ XFORM = text('xform') PROJECT = text('project') diff --git a/onadata/apps/messaging/filters.py b/onadata/apps/messaging/filters.py index 28e730d3bc..8250e2d703 100644 --- a/onadata/apps/messaging/filters.py +++ b/onadata/apps/messaging/filters.py @@ -6,7 +6,7 @@ from actstream.models import Action from django.contrib.auth.models import User -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import exceptions, filters from django_filters import rest_framework as rest_filters diff --git a/onadata/apps/messaging/serializers.py b/onadata/apps/messaging/serializers.py index c74866b936..ca71d4942e 100644 --- a/onadata/apps/messaging/serializers.py +++ b/onadata/apps/messaging/serializers.py @@ -14,7 +14,7 @@ from django.conf import settings from django.contrib.auth.models import User from django.http import HttpRequest -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import exceptions, serializers from onadata.apps.messaging.constants import MESSAGE, MESSAGE_VERBS diff --git a/onadata/apps/messaging/tests/test_backends_mqtt.py b/onadata/apps/messaging/tests/test_backends_mqtt.py index 3106379977..e4d3d3176e 100644 --- a/onadata/apps/messaging/tests/test_backends_mqtt.py +++ b/onadata/apps/messaging/tests/test_backends_mqtt.py @@ -134,12 +134,12 @@ def test_mqtt_send(self, mocked): mqtt.send(instance=instance) self.assertTrue(mocked.called) args, kwargs = mocked.call_args_list[0] - self.assertEquals(mqtt.get_topic(instance), args[0]) - self.assertEquals(get_payload(instance), kwargs['payload']) - self.assertEquals('localhost', kwargs['hostname']) - self.assertEquals(8883, kwargs['port']) - self.assertEquals(0, kwargs['qos']) - self.assertEquals(False, kwargs['retain']) + self.assertEqual(mqtt.get_topic(instance), args[0]) + self.assertEqual(get_payload(instance), kwargs['payload']) + self.assertEqual('localhost', kwargs['hostname']) + self.assertEqual(8883, kwargs['port']) + self.assertEqual(0, kwargs['qos']) + self.assertEqual(False, kwargs['retain']) self.assertDictEqual( dict(ca_certs='cacert.pem', certfile='emq.pem', diff --git a/onadata/apps/messaging/urls.py b/onadata/apps/messaging/urls.py index 2c8f45acc3..2fe9163fd8 100644 --- a/onadata/apps/messaging/urls.py +++ b/onadata/apps/messaging/urls.py @@ -2,7 +2,7 @@ """ Messaging urls module. """ -from django.conf.urls import include, url +from django.urls import include, re_path from rest_framework import routers from onadata.apps.messaging.viewsets import MessagingViewSet @@ -11,5 +11,5 @@ router.register(r'messaging', MessagingViewSet) urlpatterns = [ # pylint: disable=C0103 - url(r'^api/v1/', include(router.urls)), + re_path(r'^api/v1/', include(router.urls)), ] diff --git a/onadata/apps/restservice/RestServiceInterface.py b/onadata/apps/restservice/RestServiceInterface.py index f7d995d5fd..dca06ec26e 100644 --- a/onadata/apps/restservice/RestServiceInterface.py +++ b/onadata/apps/restservice/RestServiceInterface.py @@ -1,3 +1,12 @@ -class RestServiceInterface(object): +# -*- coding: utf-8 -*- +""" +Base class. +""" + + +class RestServiceInterface: + """RestServiceInterface base class.""" + def send(self, url, data=None): + """The class method to implement when sending data.""" raise NotImplementedError diff --git a/onadata/apps/restservice/__init__.py b/onadata/apps/restservice/__init__.py index 267a9cbd4a..59ca345a6e 100644 --- a/onadata/apps/restservice/__init__.py +++ b/onadata/apps/restservice/__init__.py @@ -1,4 +1,12 @@ -SERVICE_CHOICES = ((u'f2dhis2', u'f2dhis2'), (u'generic_json', u'JSON POST'), - (u'generic_xml', u'XML POST'), (u'bamboo', u'bamboo'), - (u'textit', u'TextIt POST'), - (u'google_sheets', u'Google Sheet')) +# -*- coding: utf-8 -*- +""" +restservice module. +""" +SERVICE_CHOICES = ( + ("f2dhis2", "f2dhis2"), + ("generic_json", "JSON POST"), + ("generic_xml", "XML POST"), + ("bamboo", "bamboo"), + ("textit", "TextIt POST"), + ("google_sheets", "Google Sheet"), +) diff --git a/onadata/apps/restservice/forms.py b/onadata/apps/restservice/forms.py index 52ff46471d..19c088347f 100644 --- a/onadata/apps/restservice/forms.py +++ b/onadata/apps/restservice/forms.py @@ -1,11 +1,19 @@ +# -*- coding: utf-8 -*- +""" +restservice forms. +""" from django import forms -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.restservice import SERVICE_CHOICES class RestServiceForm(forms.Form): - service_name = \ - forms.CharField(max_length=50, label=ugettext_lazy(u"Service Name"), - widget=forms.Select(choices=SERVICE_CHOICES)) - service_url = forms.URLField(label=ugettext_lazy(u"Service URL")) + """RestService form class.""" + + service_name = forms.CharField( + max_length=50, + label=gettext_lazy("Service Name"), + widget=forms.Select(choices=SERVICE_CHOICES), + ) + service_url = forms.URLField(label=gettext_lazy("Service URL")) diff --git a/onadata/apps/restservice/management/commands/textit_v1_to_v2.py b/onadata/apps/restservice/management/commands/textit_v1_to_v2.py index e3fcb79f09..0fddffaac9 100644 --- a/onadata/apps/restservice/management/commands/textit_v1_to_v2.py +++ b/onadata/apps/restservice/management/commands/textit_v1_to_v2.py @@ -1,42 +1,51 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +textit_v1_to_v2 - converts RapidPro/textit urls from v1 to v2 urls. +""" import re from django.core.management.base import BaseCommand -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from onadata.apps.restservice.models import RestService from onadata.libs.utils.common_tags import TEXTIT class Command(BaseCommand): + """Migrate TextIt/RapidPro v1 URLS to v2 URLS.""" + help = _("Migrate TextIt/RapidPro v1 URLS to v2 URLS.") def add_arguments(self, parser): parser.add_argument( - '--apply', default=False, help=_("Apply changes to database.")) + "--apply", default=False, help=_("Apply changes to database.") + ) + # pylint: disable=unused-argument def handle(self, *args, **options): + """Migrate TextIt/RapidPro v1 URLS to v2 URLS.""" services = RestService.objects.filter(name=TEXTIT) - force = options.get('apply') - if force and force.lower() != 'true': + force = options.get("apply") + if force and force.lower() != "true": self.stderr.write("--apply expects 'true' as a parameter value.") - - return - - v1 = re.compile(r'\/v1/runs') - v2 = '/v2/flow_starts' - - for service in services: - if v1.findall(service.service_url): - original = service.service_url - new_uri = re.sub(v1, v2, service.service_url) - params = {'v1_url': original, 'v2_url': new_uri} - if force.lower() == 'true': - service.service_url = new_uri - service.save() - self.stdout.write( - _("Changed %(v1_url)s to %(v2_url)s" % params)) - else: - self.stdout.write( - _("Will change %(v1_url)s to %(v2_url)s" % params)) + else: + version_1 = re.compile(r"\/v1/runs") + version_2 = "/v2/flow_starts" + + for service in services: + if version_1.findall(service.service_url): + original = service.service_url + new_uri = re.sub(version_1, version_2, service.service_url) + params = {"v1_url": original, "v2_url": new_uri} + if force.lower() == "true": + service.service_url = new_uri + service.save() + self.stdout.write( + _("Changed %(v1_url)s to %(v2_url)s" % params) + ) + else: + self.stdout.write( + _("Will change %(v1_url)s to %(v2_url)s" % params) + ) diff --git a/onadata/apps/restservice/models.py b/onadata/apps/restservice/models.py index 95be00081c..bcafa85f61 100644 --- a/onadata/apps/restservice/models.py +++ b/onadata/apps/restservice/models.py @@ -3,12 +3,11 @@ RestService model """ import importlib -from future.utils import python_2_unicode_compatible from django.conf import settings from django.db import models from django.db.models.signals import post_delete, post_save -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models.xform import XForm from onadata.apps.main.models import MetaData @@ -16,37 +15,38 @@ from onadata.libs.utils.common_tags import GOOGLE_SHEET, TEXTIT -@python_2_unicode_compatible class RestService(models.Model): """ Properties for an external service. """ class Meta: - app_label = 'restservice' - unique_together = ('service_url', 'xform', 'name') + app_label = "restservice" + unique_together = ("service_url", "xform", "name") - service_url = models.URLField(ugettext_lazy("Service URL")) + service_url = models.URLField(gettext_lazy("Service URL")) xform = models.ForeignKey(XForm, on_delete=models.CASCADE) name = models.CharField(max_length=50, choices=SERVICE_CHOICES) - date_created = models.DateTimeField( - auto_now_add=True, null=True, blank=True) + date_created = models.DateTimeField(auto_now_add=True, null=True, blank=True) date_modified = models.DateTimeField(auto_now=True, null=True, blank=True) - active = models.BooleanField(ugettext_lazy("Active"), default=True, - blank=False, null=False) - inactive_reason = models.TextField(ugettext_lazy("Inactive reason"), - blank=True, default="") + active = models.BooleanField( + gettext_lazy("Active"), default=True, blank=False, null=False + ) + inactive_reason = models.TextField( + gettext_lazy("Inactive reason"), blank=True, default="" + ) def __str__(self): - return u"%s:%s - %s" % (self.xform, self.long_name, self.service_url) + return f"{self.xform}:{self.long_name} - {self.service_url}" def get_service_definition(self): """ Returns ServiceDefinition class """ - services_to_modules = getattr(settings, 'REST_SERVICES_TO_MODULES', {}) + services_to_modules = getattr(settings, "REST_SERVICES_TO_MODULES", {}) module_name = services_to_modules.get( - self.name, 'onadata.apps.restservice.services.%s' % self.name) + self.name, f"onadata.apps.restservice.services.{self.name}" + ) module = importlib.import_module(module_name) return module.ServiceDefinition @@ -61,41 +61,40 @@ def long_name(self): return service_definition.verbose_name -def delete_metadata(sender, instance, **kwargs): # pylint: disable=W0613 +def delete_metadata(sender, instance, **kwargs): # pylint: disable=unused-argument """ Delete related metadata on deletion of the RestService. """ if instance.name in [TEXTIT, GOOGLE_SHEET]: MetaData.objects.filter( # pylint: disable=no-member - object_id=instance.xform.id, - data_type=instance.name).delete() + object_id=instance.xform.id, data_type=instance.name + ).delete() -post_delete.connect( - delete_metadata, sender=RestService, dispatch_uid='delete_metadata') +post_delete.connect(delete_metadata, sender=RestService, dispatch_uid="delete_metadata") -# pylint: disable=W0613 +# pylint: disable=unused-argument def propagate_merged_datasets(sender, instance, **kwargs): """ Propagate the service to the individual forms of a merged dataset. """ - created = kwargs.get('created') + created = kwargs.get("created") if created and instance.xform.is_merged_dataset: for xform in instance.xform.mergedxform.xforms.all(): RestService.objects.create( - service_url=instance.service_url, - xform=xform, - name=instance.name) + service_url=instance.service_url, xform=xform, name=instance.name + ) post_save.connect( propagate_merged_datasets, sender=RestService, - dispatch_uid='propagate_merged_datasets') + dispatch_uid="propagate_merged_datasets", +) -# pylint: disable=W0613 +# pylint: disable=unused-argument def delete_merged_datasets_service(sender, instance, **kwargs): """ Delete the service to the individual forms of a merged dataset. @@ -104,9 +103,8 @@ def delete_merged_datasets_service(sender, instance, **kwargs): for xform in instance.xform.mergedxform.xforms.all(): try: service = RestService.objects.get( - service_url=instance.service_url, - xform=xform, - name=instance.name) + service_url=instance.service_url, xform=xform, name=instance.name + ) except RestService.DoesNotExist: pass else: @@ -116,4 +114,5 @@ def delete_merged_datasets_service(sender, instance, **kwargs): post_delete.connect( delete_merged_datasets_service, sender=RestService, - dispatch_uid='propagate_merged_datasets') + dispatch_uid="propagate_merged_datasets", +) diff --git a/onadata/apps/restservice/services/f2dhis2.py b/onadata/apps/restservice/services/f2dhis2.py index eb62f473d2..c80171c04a 100644 --- a/onadata/apps/restservice/services/f2dhis2.py +++ b/onadata/apps/restservice/services/f2dhis2.py @@ -1,16 +1,25 @@ +# -*- coding: utf-8 -*- +""" +Formhub/Ona Data to DHIS2 service - push submissions to DHIS2 instance. +""" import requests from onadata.apps.restservice.RestServiceInterface import RestServiceInterface class ServiceDefinition(RestServiceInterface): - id = u'f2dhis2' - verbose_name = u'Formhub to DHIS2' + """Post submission to DHIS2 instance.""" - def send(self, url, submission_instance): - info = { - "id_string": submission_instance.xform.id_string, - "uuid": submission_instance.uuid - } - valid_url = url % info - requests.get(valid_url) + # pylint: disable=invalid-name + id = "f2dhis2" + verbose_name = "Formhub to DHIS2" + + def send(self, url, data=None): + """Post submission to DHIS2 instance.""" + if data: + info = { + "id_string": data.xform.id_string, + "uuid": data.uuid, + } + valid_url = url % info + requests.get(valid_url) diff --git a/onadata/apps/restservice/services/generic_json.py b/onadata/apps/restservice/services/generic_json.py index b1fa30c428..24c97e693d 100644 --- a/onadata/apps/restservice/services/generic_json.py +++ b/onadata/apps/restservice/services/generic_json.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Post submisison JSON data to an external service that accepts a JSON post. +""" import json import requests @@ -6,10 +10,15 @@ class ServiceDefinition(RestServiceInterface): - id = u'json' - verbose_name = u'JSON POST' + """Post submisison JSON data to an external service that accepts a JSON post.""" - def send(self, url, submission_instance): - post_data = json.dumps(submission_instance.json) - headers = {"Content-Type": "application/json"} - requests.post(url, headers=headers, data=post_data) + # pylint: disable=invalid-name + id = "json" + verbose_name = "JSON POST" + + def send(self, url, data=None): + """Post submisison JSON data to an external service that accepts a JSON post.""" + if data: + post_data = json.dumps(data.json) + headers = {"Content-Type": "application/json"} + requests.post(url, headers=headers, data=post_data) diff --git a/onadata/apps/restservice/services/generic_xml.py b/onadata/apps/restservice/services/generic_xml.py index 9a9fb212df..2ac3104cd4 100644 --- a/onadata/apps/restservice/services/generic_xml.py +++ b/onadata/apps/restservice/services/generic_xml.py @@ -1,12 +1,24 @@ +# -*- coding: utf-8 -*- +""" +Post submisison XML data to an external service that accepts an XML post. +""" import requests from onadata.apps.restservice.RestServiceInterface import RestServiceInterface class ServiceDefinition(RestServiceInterface): - id = u'xml' - verbose_name = u'XML POST' + """ + Post submisison XML data to an external service that accepts an XML post. + """ - def send(self, url, submission_instance): + # pylint: disable=invalid-name + id = "xml" + verbose_name = "XML POST" + + def send(self, url, data=None): + """ + Post submisison XML data to an external service that accepts an XML post. + """ headers = {"Content-Type": "application/xml"} - requests.post(url, data=submission_instance.xml, headers=headers) + requests.post(url, data=data.xml, headers=headers) diff --git a/onadata/apps/restservice/services/textit.py b/onadata/apps/restservice/services/textit.py index 3147e5075a..5e6333dd18 100644 --- a/onadata/apps/restservice/services/textit.py +++ b/onadata/apps/restservice/services/textit.py @@ -1,6 +1,10 @@ +# -*- coding: utf-8 -*- +""" +Post submission data to a textit/rapidpro server. +""" import json import requests -from future.utils import iteritems +from six import iteritems from six import string_types from onadata.apps.main.models import MetaData @@ -10,29 +14,36 @@ class ServiceDefinition(RestServiceInterface): + """ + Post submission data to a textit/rapidpro server. + """ + + # pylint: disable=invalid-name id = TEXTIT - verbose_name = u'TextIt POST' + verbose_name = "TextIt POST" - def send(self, url, submission_instance): + def send(self, url, data=None): """ Sends the submission to the configured rest service :param url: - :param submission_instance: + :param data: :return: """ - extra_data = self.clean_keys_of_slashes(submission_instance.json) + extra_data = self.clean_keys_of_slashes(data.json) - data_value = MetaData.textit(submission_instance.xform) + data_value = MetaData.textit(data.xform) if data_value: token, flow, contacts = data_value.split(METADATA_SEPARATOR) post_data = { "extra": extra_data, "flow": flow, - "contacts": contacts.split(',') + "contacts": contacts.split(","), + } + headers = { + "Content-Type": "application/json", + "Authorization": f"Token {token}", } - headers = {"Content-Type": "application/json", - "Authorization": "Token {}".format(token)} requests.post(url, headers=headers, data=json.dumps(post_data)) @@ -48,14 +59,12 @@ def clean_keys_of_slashes(self, record): if not isinstance(value, string_types): record[key] = str(value) - if '/' in key: + if "/" in key: # replace with _ - record[key.replace('/', '_')]\ - = record.pop(key) + record[key.replace("/", "_")] = record.pop(key) # Check if the value is a list containing nested dict and apply # same - if value and isinstance(value, list)\ - and isinstance(value[0], dict): + if value and isinstance(value, list) and isinstance(value[0], dict): for v in value: self.clean_keys_of_slashes(v) diff --git a/onadata/apps/restservice/signals.py b/onadata/apps/restservice/signals.py index c15d3fb67d..fea296a59d 100644 --- a/onadata/apps/restservice/signals.py +++ b/onadata/apps/restservice/signals.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ RestService signals module """ @@ -7,19 +7,23 @@ from onadata.apps.restservice.tasks import call_service_async -ASYNC_POST_SUBMISSION_PROCESSING_ENABLED = \ - getattr(settings, 'ASYNC_POST_SUBMISSION_PROCESSING_ENABLED', False) +ASYNC_POST_SUBMISSION_PROCESSING_ENABLED = getattr( + settings, "ASYNC_POST_SUBMISSION_PROCESSING_ENABLED", False +) -# pylint: disable=C0103 -trigger_webhook = django.dispatch.Signal(providing_args=['instance']) +# pylint: disable=invalid-name +trigger_webhook = django.dispatch.Signal(providing_args=["instance"]) -def call_webhooks(sender, **kwargs): # pylint: disable=W0613 +def call_webhooks(sender, **kwargs): # pylint: disable=unused-argument """ Call webhooks signal. """ - instance_id = kwargs['instance'].pk - call_service_async.apply_async(args=[instance_id], countdown=1) + instance_id = kwargs["instance"].pk + if ASYNC_POST_SUBMISSION_PROCESSING_ENABLED: + call_service_async.apply_async(args=[instance_id], countdown=1) + else: + call_service_async(instance_id) -trigger_webhook.connect(call_webhooks, dispatch_uid='call_webhooks') +trigger_webhook.connect(call_webhooks, dispatch_uid="call_webhooks") diff --git a/onadata/apps/restservice/tasks.py b/onadata/apps/restservice/tasks.py index b1895cc2bf..e9acd263d2 100644 --- a/onadata/apps/restservice/tasks.py +++ b/onadata/apps/restservice/tasks.py @@ -1,11 +1,16 @@ +# -*- coding: utf-8 -*- +""" +restservice async functions. +""" +from onadata.apps.logger.models.instance import Instance from onadata.apps.restservice.utils import call_service from onadata.celery import app @app.task() def call_service_async(instance_pk): + """Async function that calls call_service().""" # load the parsed instance - from onadata.apps.logger.models.instance import Instance try: instance = Instance.objects.get(pk=instance_pk) diff --git a/onadata/apps/restservice/tests/test_restservice.py b/onadata/apps/restservice/tests/test_restservice.py index a531d2577a..21ea036b34 100644 --- a/onadata/apps/restservice/tests/test_restservice.py +++ b/onadata/apps/restservice/tests/test_restservice.py @@ -1,117 +1,143 @@ +# -*- coding: utf-8 -*- +""" +Test RestService model +""" import os import time from django.test.utils import override_settings from django.urls import reverse + from mock import patch from onadata.apps.logger.models.xform import XForm from onadata.apps.main.models import MetaData from onadata.apps.main.tests.test_base import TestBase from onadata.apps.main.views import show -from onadata.apps.restservice.RestServiceInterface import RestServiceInterface from onadata.apps.restservice.models import RestService +from onadata.apps.restservice.RestServiceInterface import RestServiceInterface from onadata.apps.restservice.services.textit import ServiceDefinition from onadata.apps.restservice.views import add_service, delete_service class RestServiceTest(TestBase): + """ + Test RestService model + """ def setUp(self): - self.service_url = u'http://0.0.0.0:8001/%(id_string)s/post/%(uuid)s' - self.service_name = u'f2dhis2' + self.service_url = "http://0.0.0.0:8001/%(id_string)s/post/%(uuid)s" + self.service_name = "f2dhis2" self._create_user_and_login() - filename = u'dhisform.xlsx' + filename = "dhisform.xlsx" self.this_directory = os.path.dirname(__file__) - path = os.path.join(self.this_directory, u'fixtures', filename) + path = os.path.join(self.this_directory, "fixtures", filename) self._publish_xls_file(path) self.xform = XForm.objects.all().reverse()[0] - def wait(self, t=1): - time.sleep(t) + # pylint: disable=no-self-use + def wait(self, duration=1): + """Sleep for 1 second or as defined by ``duration``.""" + time.sleep(duration) def _create_rest_service(self): - rs = RestService(service_url=self.service_url, - xform=self.xform, name=self.service_name) - rs.save() - self.restservice = rs + service = RestService( + service_url=self.service_url, xform=self.xform, name=self.service_name + ) + service.save() + + return service def _add_rest_service(self, service_url, service_name): - add_service_url = reverse(add_service, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - }) + add_service_url = reverse( + add_service, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.client.get(add_service_url, {}) count = RestService.objects.all().count() self.assertEqual(response.status_code, 200) - post_data = {'service_url': service_url, - 'service_name': service_name} + post_data = {"service_url": service_url, "service_name": service_name} response = self.client.post(add_service_url, post_data) self.assertEqual(response.status_code, 200) - self.assertEquals(RestService.objects.all().count(), count + 1) - - def add_rest_service_with_usename_and_id_string_in_uppercase(self): - add_service_url = reverse(add_service, kwargs={ - 'username': self.user.username.upper(), - 'id_string': self.xform.id_string.upper() - }) + self.assertEqual(RestService.objects.all().count(), count + 1) + + # pylint: disable=invalid-name + def add_rest_service_with_username_and_id_string_in_uppercase(self): + """Test that the service url is not case sensitive""" + add_service_url = reverse( + add_service, + kwargs={ + "username": self.user.username.upper(), + "id_string": self.xform.id_string.upper(), + }, + ) response = self.client.get(add_service_url, {}) self.assertEqual(response.status_code, 200) def test_create_rest_service(self): + """Test the RestService model.""" count = RestService.objects.all().count() self._create_rest_service() - self.assertEquals(RestService.objects.all().count(), count + 1) + self.assertEqual(RestService.objects.all().count(), count + 1) def test_service_definition(self): - self._create_rest_service() - sv = self.restservice.get_service_definition()() - self.assertEqual(isinstance(sv, RestServiceInterface), True) + """Test the service_definition is an instance of RestServiceInterface""" + restservice = self._create_rest_service() + service = restservice.get_service_definition()() + self.assertEqual(isinstance(service, RestServiceInterface), True) def test_add_service(self): + """Test adding a restservice.""" self._add_rest_service(self.service_url, self.service_name) def test_anon_service_view(self): + """Test the rest service section is not available to asynchronous users.""" self.xform.shared = True self.xform.save() self._logout() - url = reverse(show, kwargs={ - 'username': self.xform.user.username, - 'id_string': self.xform.id_string - }) + url = reverse( + show, + kwargs={ + "username": self.xform.user.username, + "id_string": self.xform.id_string, + }, + ) response = self.client.get(url) self.assertNotContains( response, '

Rest Services

') + '"#restservice_tab">Rest Services', + ) def test_delete_service(self): + """Test deletion of a service.""" self._add_rest_service(self.service_url, self.service_name) count = RestService.objects.all().count() service = RestService.objects.reverse()[0] - post_data = {'service-id': service.pk} - del_service_url = reverse(delete_service, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - }) + post_data = {"service-id": service.pk} + del_service_url = reverse( + delete_service, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.client.post(del_service_url, post_data) self.assertEqual(response.status_code, 200) - self.assertEqual( - RestService.objects.all().count(), count - 1 - ) + self.assertEqual(RestService.objects.all().count(), count - 1) + # pylint: disable=invalid-name def test_add_rest_service_with_wrong_id_string(self): - add_service_url = reverse(add_service, kwargs={ - 'username': self.user.username, - 'id_string': 'wrong_id_string'}) - post_data = {'service_url': self.service_url, - 'service_name': self.service_name} + """Test the id_string is validated when adding a service url.""" + add_service_url = reverse( + add_service, + kwargs={"username": self.user.username, "id_string": "wrong_id_string"}, + ) + post_data = {"service_url": self.service_url, "service_name": self.service_name} response = self.client.post(add_service_url, post_data) self.assertEqual(response.status_code, 404) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch('requests.post') + @patch("requests.post") def test_textit_service(self, mock_http): + """Test the textit service.""" service_url = "https://textit.io/api/v1/runs.json" service_name = "textit" @@ -123,46 +149,47 @@ def test_textit_service(self, mock_http): default_contact = "sadlsdfskjdfds" MetaData.textit( - self.xform, data_value="{}|{}|{}".format(api_token, - flow_uuid, - default_contact)) + self.xform, + data_value=f"{api_token}|{flow_uuid}|{default_contact}", + ) - xml_submission = os.path.join(self.this_directory, - u'fixtures', - u'dhisform_submission1.xml') + xml_submission = os.path.join( + self.this_directory, "fixtures", "dhisform_submission1.xml" + ) self.assertFalse(mock_http.called) self._make_submission(xml_submission) self.assertTrue(mock_http.called) - self.assertEquals(mock_http.call_count, 1) + self.assertEqual(mock_http.call_count, 1) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch('requests.post') + @patch("requests.post") def test_rest_service_not_set(self, mock_http): - xml_submission = os.path.join(self.this_directory, - u'fixtures', - u'dhisform_submission1.xml') + """Test a requests.post is not called when a service is not defined.""" + xml_submission = os.path.join( + self.this_directory, "fixtures", "dhisform_submission1.xml" + ) self.assertFalse(mock_http.called) self._make_submission(xml_submission) self.assertFalse(mock_http.called) - self.assertEquals(mock_http.call_count, 0) + self.assertEqual(mock_http.call_count, 0) def test_clean_keys_of_slashes(self): + """Test ServiceDefinition.clean_keys_of_slashes() function.""" service = ServiceDefinition() test_data = { "hh/group/data_set": "22", "empty_column": "", "false_column": False, - "zero_column": 0 + "zero_column": 0, } expected_data = { "hh_group_data_set": "22", "false_column": "False", - "zero_column": "0" + "zero_column": "0", } - self.assertEquals(expected_data, - service.clean_keys_of_slashes(test_data)) + self.assertEqual(expected_data, service.clean_keys_of_slashes(test_data)) diff --git a/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py b/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py index 0fd8b1536f..19ad06421a 100644 --- a/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py +++ b/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py @@ -1,51 +1,62 @@ +# -*- coding: utf-8 -*- +""" +Test /restservices API endpoint implementation. +""" from django.test.utils import override_settings + from mock import patch -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.main.models.meta_data import MetaData from onadata.apps.restservice.models import RestService -from onadata.apps.restservice.viewsets.restservices_viewset import \ - RestServicesViewSet +from onadata.apps.restservice.viewsets.restservices_viewset import RestServicesViewSet class TestRestServicesViewSet(TestAbstractViewSet): + """ + Test /restservices API endpoint implementation. + """ def setUp(self): - super(TestRestServicesViewSet, self).setUp() - self.view = RestServicesViewSet.as_view({ - 'delete': 'destroy', - 'get': 'retrieve', - 'post': 'create', - 'put': 'update', - 'patch': 'partial_update' - }) + super().setUp() + self.view = RestServicesViewSet.as_view( + { + "delete": "destroy", + "get": "retrieve", + "post": "create", + "put": "update", + "patch": "partial_update", + } + ) self._publish_xls_form_to_project() def test_create(self): + """Test create service via API.""" count = RestService.objects.all().count() post_data = { "name": "generic_json", "service_url": "https://textit.io", - "xform": self.xform.pk + "xform": self.xform.pk, } - request = self.factory.post('/', data=post_data, **self.extra) + request = self.factory.post("/", data=post_data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 201) - self.assertEquals(count + 1, RestService.objects.all().count()) + self.assertEqual(response.status_code, 201) + self.assertEqual(count + 1, RestService.objects.all().count()) + # pylint: disable=invalid-name def test_textit_service_missing_params(self): + """Test creating a service with a missing parameter fails.""" post_data = { "name": "textit", "service_url": "https://textit.io", "xform": self.xform.pk, } - request = self.factory.post('/', data=post_data, **self.extra) + request = self.factory.post("/", data=post_data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) + self.assertEqual(response.status_code, 400) def _create_textit_service(self): count = RestService.objects.all().count() @@ -56,84 +67,89 @@ def _create_textit_service(self): "xform": self.xform.pk, "auth_token": "sadsdfhsdf", "flow_uuid": "sdfskhfskdjhfs", - "contacts": "ksadaskjdajsda" + "contacts": "ksadaskjdajsda", } - request = self.factory.post('/', data=post_data, **self.extra) + request = self.factory.post("/", data=post_data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 201) - self.assertEquals(count + 1, RestService.objects.all().count()) + self.assertEqual(response.status_code, 201) + self.assertEqual(count + 1, RestService.objects.all().count()) - meta = MetaData.objects.filter(object_id=self.xform.id, - data_type='textit') - self.assertEquals(len(meta), 1) - rs = RestService.objects.last() + meta = MetaData.objects.filter(object_id=self.xform.id, data_type="textit") + self.assertEqual(len(meta), 1) + service = RestService.objects.last() expected_dict = { - 'name': u'textit', - 'contacts': u'ksadaskjdajsda', - 'auth_token': u'sadsdfhsdf', - 'flow_uuid': u'sdfskhfskdjhfs', - 'service_url': u'https://textit.io', - 'id': rs.pk, - 'xform': self.xform.pk, - 'active': True, - 'inactive_reason': '', - 'flow_title': '' + "name": "textit", + "contacts": "ksadaskjdajsda", + "auth_token": "sadsdfhsdf", + "flow_uuid": "sdfskhfskdjhfs", + "service_url": "https://textit.io", + "id": service.pk, + "xform": self.xform.pk, + "active": True, + "inactive_reason": "", + "flow_title": "", } - response.data.pop('date_modified') - response.data.pop('date_created') + response.data.pop("date_modified") + response.data.pop("date_created") self.assertEqual(response.data, expected_dict) return response.data def test_create_textit_service(self): + """Test creating textit service via API.""" self._create_textit_service() def test_retrieve_textit_services(self): + """Test retrieving the textit service via API.""" response_data = self._create_textit_service() - _id = response_data.get('id') + _id = response_data.get("id") - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=_id) expected_dict = { - 'name': u'textit', - 'contacts': u'ksadaskjdajsda', - 'auth_token': u'sadsdfhsdf', - 'flow_uuid': u'sdfskhfskdjhfs', - 'service_url': u'https://textit.io', - 'id': _id, - 'xform': self.xform.pk, - 'active': True, - 'inactive_reason': '', - 'flow_title': '' + "name": "textit", + "contacts": "ksadaskjdajsda", + "auth_token": "sadsdfhsdf", + "flow_uuid": "sdfskhfskdjhfs", + "service_url": "https://textit.io", + "id": _id, + "xform": self.xform.pk, + "active": True, + "inactive_reason": "", + "flow_title": "", } - response.data.pop('date_modified') - response.data.pop('date_created') + response.data.pop("date_modified") + response.data.pop("date_created") self.assertEqual(response.data, expected_dict) def test_delete_textit_service(self): + """Test deleting a textit service via API""" rest = self._create_textit_service() count = RestService.objects.all().count() - meta_count = MetaData.objects.filter(object_id=self.xform.id, - data_type='textit').count() - - request = self.factory.delete('/', **self.extra) - response = self.view(request, pk=rest['id']) - - self.assertEquals(response.status_code, 204) - self.assertEquals(count - 1, RestService.objects.all().count()) - a_meta_count = MetaData.objects.filter(object_id=self.xform.id, - data_type='textit').count() + meta_count = MetaData.objects.filter( + object_id=self.xform.id, data_type="textit" + ).count() + + request = self.factory.delete("/", **self.extra) + response = self.view(request, pk=rest["id"]) + + self.assertEqual(response.status_code, 204) + self.assertEqual(count - 1, RestService.objects.all().count()) + a_meta_count = MetaData.objects.filter( + object_id=self.xform.id, data_type="textit" + ).count() self.assertEqual(meta_count - 1, a_meta_count) def test_update(self): - rest = RestService(name="testservice", - service_url="http://serviec.io", - xform=self.xform) + """Test updating a service via API.""" + rest = RestService( + name="testservice", service_url="http://serviec.io", xform=self.xform + ) rest.save() post_data = { @@ -143,48 +159,53 @@ def test_update(self): "auth_token": "sadsdfhsdf", "flow_uuid": "sdfskhfskdjhfs", "contacts": "ksadaskjdajsda", - "flow_title": "test-flow" + "flow_title": "test-flow", } - request = self.factory.put('/', data=post_data, **self.extra) + request = self.factory.put("/", data=post_data, **self.extra) response = self.view(request, pk=rest.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['name'], "textit") - self.assertEqual(response.data['flow_title'], 'test-flow') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["name"], "textit") + self.assertEqual(response.data["flow_title"], "test-flow") metadata_count = MetaData.objects.count() # Flow title can be updated put_data = { - 'flow_title': 'new-name', - 'xform': self.xform.pk, - 'name': 'textit', - 'service_url': 'https://textit.io', - 'auth_token': 'sadsdfhsdf', - 'flow_uuid': 'sdfskhfskdjhfs', - 'contacts': 'ksadaskjdajsda', + "flow_title": "new-name", + "xform": self.xform.pk, + "name": "textit", + "service_url": "https://textit.io", + "auth_token": "sadsdfhsdf", + "flow_uuid": "sdfskhfskdjhfs", + "contacts": "ksadaskjdajsda", } - request = self.factory.put('/', data=put_data, **self.extra) + request = self.factory.put("/", data=put_data, **self.extra) response = self.view(request, pk=rest.pk) self.assertEqual(response.status_code, 200, response.data) - self.assertEqual(response.data['flow_title'], 'new-name') + self.assertEqual(response.data["flow_title"], "new-name") self.assertEqual(MetaData.objects.count(), metadata_count) def test_update_with_errors(self): + """Test update errors if records is not in the write format.""" rest = self._create_textit_service() - data_value = "{}|{}".format("test", "test2") + data_value = "test|test2" MetaData.textit(self.xform, data_value) - request = self.factory.get('/', **self.extra) - response = self.view(request, pk=rest.get('id')) + request = self.factory.get("/", **self.extra) + response = self.view(request, pk=rest.get("id")) - self.assertEquals(response.status_code, 400) - self.assertEquals(response.data, - [u"Error occurred when loading textit service." - u"Resolve by updating auth_token, flow_uuid and " - u"contacts fields"]) + self.assertEqual(response.status_code, 400) + self.assertEqual( + response.data, + [ + "Error occurred when loading textit service." + "Resolve by updating auth_token, flow_uuid and " + "contacts fields" + ], + ) post_data = { "name": "textit", @@ -192,64 +213,67 @@ def test_update_with_errors(self): "xform": self.xform.pk, "auth_token": "sadsdfhsdf", "flow_uuid": "sdfskhfskdjhfs", - "contacts": "ksadaskjdajsda" + "contacts": "ksadaskjdajsda", } - request = self.factory.put('/', data=post_data, **self.extra) - response = self.view(request, pk=rest.get('id')) + request = self.factory.put("/", data=post_data, **self.extra) + response = self.view(request, pk=rest.get("id")) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) def test_delete(self): - rest = RestService(name="testservice", - service_url="http://serviec.io", - xform=self.xform) + """Test delete service via API.""" + rest = RestService( + name="testservice", service_url="http://serviec.io", xform=self.xform + ) rest.save() count = RestService.objects.all().count() - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = self.view(request, pk=rest.pk) - self.assertEquals(response.status_code, 204) - self.assertEquals(count - 1, RestService.objects.all().count()) + self.assertEqual(response.status_code, 204) + self.assertEqual(count - 1, RestService.objects.all().count()) def test_retrieve(self): - rest = RestService(name="testservice", - service_url="http://serviec.io", - xform=self.xform) + """Test retrieving a service via API.""" + rest = RestService( + name="testservice", service_url="http://serviec.io", xform=self.xform + ) rest.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=rest.pk) data = { - 'id': rest.pk, - 'xform': self.xform.pk, - 'name': u'testservice', - 'service_url': u'http://serviec.io', - 'active': True, - 'inactive_reason': '' + "id": rest.pk, + "xform": self.xform.pk, + "name": "testservice", + "service_url": "http://serviec.io", + "active": True, + "inactive_reason": "", } - response.data.pop('date_modified') - response.data.pop('date_created') - self.assertEquals(response.status_code, 200) + response.data.pop("date_modified") + response.data.pop("date_created") + self.assertEqual(response.status_code, 200) - self.assertEquals(response.data, data) + self.assertEqual(response.data, data) def test_duplicate_rest_service(self): + """Test duplicate service is not created.""" post_data = { "name": "textit", "service_url": "https://textit.io", "xform": self.xform.pk, "auth_token": "sadsdfhsdf", "flow_uuid": "sdfskhfskdjhfs", - "contacts": "ksadaskjdajsda" + "contacts": "ksadaskjdajsda", } - request = self.factory.post('/', data=post_data, **self.extra) + request = self.factory.post("/", data=post_data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 201) + self.assertEqual(response.status_code, 201) post_data = { "name": "textit", @@ -257,49 +281,35 @@ def test_duplicate_rest_service(self): "xform": self.xform.pk, "auth_token": "sadsdfhsdf", "flow_uuid": "sdfskhfskdjhfs", - "contacts": "ksadaskjdajsda" + "contacts": "ksadaskjdajsda", } - request = self.factory.post('/', data=post_data, **self.extra) + request = self.factory.post("/", data=post_data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) + self.assertEqual(response.status_code, 400) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch('requests.post') + @patch("requests.post") def test_textit_flow(self, mock_http): - rest = RestService(name="textit", - service_url="https://server.io", - xform=self.xform) + """Test posting a submission to textit service.""" + rest = RestService( + name="textit", service_url="https://server.io", xform=self.xform + ) rest.save() - MetaData.textit(self.xform, - data_value='{}|{}|{}'.format("sadsdfhsdf", - "sdfskhfskdjhfs", - "ksadaskjdajsda")) + MetaData.textit( + self.xform, + data_value="sadsdfhsdf|sdfskhfskdjhfs|ksadaskjdajsda", + ) self.assertFalse(mock_http.called) self._make_submissions() self.assertTrue(mock_http.called) - self.assertEquals(mock_http.call_count, 4) - - @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch('requests.post') - def test_textit_flow_without_parsed_instances(self, mock_http): - rest = RestService(name="textit", - service_url="https://server.io", - xform=self.xform) - rest.save() - - MetaData.textit(self.xform, - data_value='{}|{}|{}'.format("sadsdfhsdf", - "sdfskhfskdjhfs", - "ksadaskjdajsda")) - self.assertFalse(mock_http.called) - self._make_submissions() - self.assertTrue(mock_http.called) + self.assertEqual(mock_http.call_count, 4) def test_create_rest_service_invalid_form_id(self): + """Test creating with an invalid form id fails.""" count = RestService.objects.all().count() post_data = { @@ -308,11 +318,11 @@ def test_create_rest_service_invalid_form_id(self): "xform": "invalid", "auth_token": "sadsdfhsdf", "flow_uuid": "sdfskhfskdjhfs", - "contacts": "ksadaskjdajsda" + "contacts": "ksadaskjdajsda", } - request = self.factory.post('/', data=post_data, **self.extra) + request = self.factory.post("/", data=post_data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertEqual(response.data, {'xform': [u'Invalid form id']}) - self.assertEquals(count, RestService.objects.all().count()) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.data, {"xform": ["Invalid form id"]}) + self.assertEqual(count, RestService.objects.all().count()) diff --git a/onadata/apps/restservice/utils.py b/onadata/apps/restservice/utils.py index a16e7061f5..795157968e 100644 --- a/onadata/apps/restservice/utils.py +++ b/onadata/apps/restservice/utils.py @@ -1,21 +1,27 @@ +# -*- coding: utf-8 -*- +""" +restservice utility functions. +""" import logging - -from django.utils.translation import ugettext as _ +import sys from onadata.apps.restservice.models import RestService from onadata.libs.utils.common_tags import GOOGLE_SHEET +from onadata.libs.utils.common_tools import report_exception def call_service(submission_instance): + """Sends submissions to linked services.""" # lookup service which is not google sheet service services = RestService.objects.filter( - xform_id=submission_instance.xform_id).exclude(name=GOOGLE_SHEET) + xform_id=submission_instance.xform_id + ).exclude(name=GOOGLE_SHEET) # call service send with url and data parameters - for sv in services: - # TODO: Queue service + for service_def in services: + # pylint: disable=broad-except try: - service = sv.get_service_definition()() - service.send(sv.service_url, submission_instance) + service = service_def.get_service_definition()() + service.send(service_def.service_url, submission_instance) except Exception as e: - # TODO: Handle gracefully | requeue/resend - logging.exception(_(u'Service threw exception: %s' % str(e))) + report_exception(f"Service call failed: {e}", e, sys.exc_info()) + logging.exception("Service threw exception: %s", e) diff --git a/onadata/apps/restservice/views.py b/onadata/apps/restservice/views.py index 4878a488f0..e8e1a4c5c3 100644 --- a/onadata/apps/restservice/views.py +++ b/onadata/apps/restservice/views.py @@ -1,87 +1,102 @@ -import json +# -*- coding: utf-8 -*- +""" +restservice views. +""" from django.contrib.auth.decorators import login_required from django.db.utils import IntegrityError -from django.http import HttpResponse +from django.http import HttpResponse, JsonResponse from django.shortcuts import render from django.template.base import Template from django.template.context import Context from django.template.loader import render_to_string -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ -from onadata.libs.utils.viewer_tools import get_form from onadata.apps.restservice.forms import RestServiceForm from onadata.apps.restservice.models import RestService +from onadata.libs.utils.viewer_tools import get_form @login_required def add_service(request, username, id_string): + """Add a service.""" data = {} form = RestServiceForm() - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } - xform = get_form(xform_kwargs) - if request.method == 'POST': + xform = get_form( + {"id_string__iexact": id_string, "user__username__iexact": username} + ) + if request.method == "POST": form = RestServiceForm(request.POST) restservice = None if form.is_valid(): - service_name = form.cleaned_data['service_name'] - service_url = form.cleaned_data['service_url'] + service_name = form.cleaned_data["service_name"] + service_url = form.cleaned_data["service_url"] try: - rs = RestService(service_url=service_url, - name=service_name, xform=xform) - rs.save() + service = RestService( + service_url=service_url, name=service_name, xform=xform + ) + service.save() except IntegrityError: - message = _(u"Service already defined.") - status = 'fail' + message = _("Service already defined.") + status = "fail" else: - status = 'success' - message = (_(u"Successfully added service %(name)s.") - % {'name': service_name}) - service_tpl = render_to_string("service.html", { - "sv": rs, "username": xform.user.username, - "id_string": xform.id_string}) + status = "success" + message = _("Successfully added service %(name)s.") % { + "name": service_name + } + service_tpl = render_to_string( + "service.html", + { + "sv": service, + "username": xform.user.username, + "id_string": xform.id_string, + }, + ) restservice = service_tpl else: - status = 'fail' - message = _(u"Please fill in all required fields") + status = "fail" + message = _("Please fill in all required fields") if form.errors: for field in form: - message += Template(u"{{ field.errors }}")\ - .render(Context({'field': field})) + message += Template("{{ field.errors }}").render( + Context({"field": field}) + ) if request.is_ajax(): - response = {'status': status, 'message': message} + response = {"status": status, "message": message} if restservice: - response["restservice"] = u"%s" % restservice + response["restservice"] = f"{restservice}" - return HttpResponse(json.dumps(response)) + return JsonResponse(response) - data['status'] = status - data['message'] = message + data["status"] = status + data["message"] = message - data['list_services'] = RestService.objects.filter(xform=xform) - data['form'] = form - data['username'] = username - data['id_string'] = id_string + data["list_services"] = RestService.objects.filter(xform=xform) + data["form"] = form + data["username"] = username + data["id_string"] = id_string return render(request, "add-service.html", data) +@login_required def delete_service(request, username, id_string): + """Delete a service view.""" success = "FAILED" - if request.method == 'POST': - pk = request.POST.get('service-id') - if pk: + xform = get_form( + {"id_string__iexact": id_string, "user__username__iexact": username} + ) + if request.method == "POST": + service_id = request.POST.get("service-id") + if service_id: try: - rs = RestService.objects.get(pk=int(pk)) + service = RestService.objects.get(pk=int(service_id), xform=xform) except RestService.DoesNotExist: pass else: - rs.delete() + service.delete() success = "OK" return HttpResponse(success) diff --git a/onadata/apps/restservice/viewsets/restservices_viewset.py b/onadata/apps/restservice/viewsets/restservices_viewset.py index 932aa8db56..b0862920c1 100644 --- a/onadata/apps/restservice/viewsets/restservices_viewset.py +++ b/onadata/apps/restservice/viewsets/restservices_viewset.py @@ -1,28 +1,31 @@ +# -*- coding: utf-8 -*- +""" +Implements the /api/v1/restservices endpoint. +""" from django.conf import settings from django.utils.module_loading import import_string -from rest_framework.viewsets import ModelViewSet + from rest_framework.response import Response +from rest_framework.viewsets import ModelViewSet from onadata.apps.api.permissions import RestServiceObjectPermissions -from onadata.libs.serializers.textit_serializer import TextItSerializer +from onadata.apps.api.tools import get_baseviewset_class from onadata.apps.restservice.models import RestService from onadata.libs import filters -from onadata.libs.serializers.restservices_serializer import \ - RestServiceSerializer -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.last_modified_mixin import LastModifiedMixin +from onadata.libs.serializers.restservices_serializer import RestServiceSerializer +from onadata.libs.serializers.textit_serializer import TextItSerializer from onadata.libs.utils.common_tags import TEXTIT -from onadata.apps.api.tools import get_baseviewset_class - +# pylint: disable=invalid-name BaseViewset = get_baseviewset_class() def get_serializer_class(name): - services_to_serializers = getattr(settings, - 'REST_SERVICES_TO_SERIALIZERS', {}) + """Returns a serilizer class with the given ``name``.""" + services_to_serializers = getattr(settings, "REST_SERVICES_TO_SERIALIZERS", {}) serializer_class = services_to_serializers.get(name) if serializer_class: @@ -31,27 +34,36 @@ def get_serializer_class(name): if name == TEXTIT: return TextItSerializer + return RestServiceSerializer + -class RestServicesViewSet(AuthenticateHeaderMixin, - CacheControlMixin, LastModifiedMixin, BaseViewset, - ModelViewSet): +# pylint: disable=too-many-ancestors +class RestServicesViewSet( + AuthenticateHeaderMixin, + CacheControlMixin, + LastModifiedMixin, + BaseViewset, + ModelViewSet, +): """ This endpoint provides access to form rest services. """ - queryset = RestService.objects.select_related('xform') + queryset = RestService.objects.select_related("xform") serializer_class = RestServiceSerializer - permission_classes = [RestServiceObjectPermissions, ] - filter_backends = (filters.RestServiceFilter, ) + permission_classes = [ + RestServiceObjectPermissions, + ] + filter_backends = (filters.RestServiceFilter,) def get_serializer_class(self): - name = self.request.data.get('name') + name = self.request.data.get("name") serializer_class = get_serializer_class(name) if serializer_class: return serializer_class - return super(RestServicesViewSet, self).get_serializer_class() + return super().get_serializer_class() def retrieve(self, request, *args, **kwargs): instance = self.get_object() diff --git a/onadata/apps/sms_support/autodoc.py b/onadata/apps/sms_support/autodoc.py index d822892019..36e959e641 100644 --- a/onadata/apps/sms_support/autodoc.py +++ b/onadata/apps/sms_support/autodoc.py @@ -13,28 +13,34 @@ from __future__ import absolute_import import datetime -import json from builtins import str as text from onadata.apps.sms_support.tools import ( - DEFAULT_ALLOW_MEDIA, DEFAULT_DATE_FORMAT, DEFAULT_DATETIME_FORMAT, - DEFAULT_SEPARATOR, MEDIA_TYPES) + DEFAULT_ALLOW_MEDIA, + DEFAULT_DATE_FORMAT, + DEFAULT_DATETIME_FORMAT, + DEFAULT_SEPARATOR, + MEDIA_TYPES, +) -def get_sample_data_for(question, json_survey, as_names=False): - """ return an example data for a particular question. +# pylint: disable=too-many-return-statements,too-many-branches +def get_sample_data_for(question, json_survey, as_names=False): # noqa C901 + """return an example data for a particular question. - If as_names is True, returns name (not sms_field) of the question """ + If as_names is True, returns name (not sms_field) of the question""" - xlsf_name = question.get('name') - xlsf_type = question.get('type') - xlsf_choices = question.get('children') + xlsf_name = question.get("name") + xlsf_type = question.get("type") + xlsf_choices = question.get("children") now = datetime.datetime.now() - xlsf_date_fmt = json_survey.get('sms_date_format', DEFAULT_DATE_FORMAT) \ - or DEFAULT_DATE_FORMAT - xlsf_datetime_fmt = json_survey.get('sms_date_format', - DEFAULT_DATETIME_FORMAT) \ + xlsf_date_fmt = ( + json_survey.get("sms_date_format", DEFAULT_DATE_FORMAT) or DEFAULT_DATE_FORMAT + ) + xlsf_datetime_fmt = ( + json_survey.get("sms_date_format", DEFAULT_DATETIME_FORMAT) or DEFAULT_DATETIME_FORMAT + ) def safe_wrap(value): return text(value) @@ -42,169 +48,175 @@ def safe_wrap(value): if as_names: return xlsf_name - if xlsf_type == 'text': - return safe_wrap('lorem ipsum') - elif xlsf_type == 'integer': + if xlsf_type == "text": + return safe_wrap("lorem ipsum") + if xlsf_type == "integer": return safe_wrap(4) - elif xlsf_type == 'decimal': + if xlsf_type == "decimal": return safe_wrap(1.2) - elif xlsf_type == 'select one': - return safe_wrap( - u' '.join([c.get('sms_option') for c in xlsf_choices][:1])) - elif xlsf_type == 'select all that apply': - return safe_wrap( - u' '.join([c.get('sms_option') for c in xlsf_choices][:2])) - elif xlsf_type == 'geopoint': - return safe_wrap('12.65 -8') - elif xlsf_type in MEDIA_TYPES: - exts = {'audio': 'mp3', 'video': 'avi', 'photo': 'jpg'} - return safe_wrap('x.%s;dGhpc' % exts.get(xlsf_type, 'ext')) - elif xlsf_type == 'barcode': - return safe_wrap('abc') - elif xlsf_type == 'date': + if xlsf_type == "select one": + return safe_wrap(" ".join([c.get("sms_option") for c in xlsf_choices][:1])) + if xlsf_type == "select all that apply": + return safe_wrap(" ".join([c.get("sms_option") for c in xlsf_choices][:2])) + if xlsf_type == "geopoint": + return safe_wrap("12.65 -8") + if xlsf_type in MEDIA_TYPES: + exts = {"audio": "mp3", "video": "avi", "photo": "jpg"} + return safe_wrap(f"x.{exts.get(xlsf_type, 'ext')};dGhpc") + if xlsf_type == "barcode": + return safe_wrap("abc") + if xlsf_type == "date": return safe_wrap(now.strftime(xlsf_date_fmt)) - elif xlsf_type == 'datetime': + if xlsf_type == "datetime": return safe_wrap(now.strftime(xlsf_datetime_fmt)) - elif xlsf_type == 'note': + if xlsf_type == "note": return None - else: - return safe_wrap('?') + return safe_wrap("?") -def get_helper_text(question, json_survey): - """ The full sentence (html) of the helper for a question +def get_helper_text(question, json_survey): # noqa C901 + """The full sentence (html) of the helper for a question - Includes the type, a description - and potentialy accepted values or format """ + Includes the type, a description + and potentialy accepted values or format""" - xlsf_type = question.get('type') - xlsf_choices = question.get('children') - xlsf_date_fmt = json_survey.get('sms_date_format', DEFAULT_DATE_FORMAT) \ - or DEFAULT_DATE_FORMAT - xlsf_datetime_fmt = json_survey.get('sms_date_format', - DEFAULT_DATETIME_FORMAT) \ + xlsf_type = question.get("type") + xlsf_choices = question.get("children") + xlsf_date_fmt = ( + json_survey.get("sms_date_format", DEFAULT_DATE_FORMAT) or DEFAULT_DATE_FORMAT + ) + xlsf_datetime_fmt = ( + json_survey.get("sms_date_format", DEFAULT_DATETIME_FORMAT) or DEFAULT_DATETIME_FORMAT - separator = json_survey.get('sms_separator', DEFAULT_SEPARATOR) \ - or DEFAULT_SEPARATOR + ) + separator = json_survey.get("sms_separator", DEFAULT_SEPARATOR) or DEFAULT_SEPARATOR def safe_wrap(value, xlsf_type=xlsf_type): value = ( - u'%(type)s ' - u'%(text)s' % { - 'type': xlsf_type, - 'text': value - }) + '%(type)s ' + '%(text)s' + % {"type": xlsf_type, "text": value} + ) return text(value) - if xlsf_type == 'text': - return safe_wrap(u'Any string (excluding “%s”)' % separator) - elif xlsf_type == 'integer': - return safe_wrap('Any integer digit.') - elif xlsf_type == 'decimal': - return safe_wrap('A decimal or integer value.') - elif xlsf_type == 'select one': - helper = u'Select one of the following:' - helper += u'
    ' - # pylint: disable=E1101 - helper += u''.join([ - u'
  • ' - u'%(sms_option)s %(label)s
    • ' % { - 'sms_option': c.get('sms_option'), - 'label': c.get('label') - } for c in xlsf_choices - ]) - helper += u'
' + if xlsf_type == "text": + return safe_wrap(f"Any string (excluding “{separator}”)") + if xlsf_type == "integer": + return safe_wrap("Any integer digit.") + if xlsf_type == "decimal": + return safe_wrap("A decimal or integer value.") + if xlsf_type == "select one": + helper = "Select one of the following:" + helper += "
    " + # pylint: disable=no-member + helper += "".join( + [ + '
  • ' + '%(sms_option)s %(label)s
    • ' + % {"sms_option": c.get("sms_option"), "label": c.get("label")} + for c in xlsf_choices + ] + ) + helper += "
" return safe_wrap(helper) - elif xlsf_type == 'select all that apply': - helper = u'Select none, one or more in:' - helper += u'
    ' - # pylint: disable=E1101 - helper += u''.join([ - u'
  • ' - u'%(sms_option)s %(label)s
    • ' % { - 'sms_option': c.get('sms_option'), - 'label': c.get('label') - } for c in xlsf_choices - ]) - helper += u'
' + if xlsf_type == "select all that apply": + helper = "Select none, one or more in:" + helper += "
    " + # pylint: disable=no-member + helper += "".join( + [ + '
  • ' + '%(sms_option)s %(label)s
    • ' + % {"sms_option": c.get("sms_option"), "label": c.get("label")} + for c in xlsf_choices + ] + ) + helper += "
" return safe_wrap(helper) - elif xlsf_type == 'geopoint': - helper = (u'GPS coordinates as ' - u'latitude longitude.' - u'
Optionnaly add ' - u'altitude precision after. All of them are decimal.') + if xlsf_type == "geopoint": + helper = ( + 'GPS coordinates as ' + "latitude longitude." + '
Optionnaly add ' + "altitude precision after. All of them are decimal." + ) return safe_wrap(helper) - elif xlsf_type in MEDIA_TYPES: - exts = {'audio': 'mp3', 'video': 'avi', 'photo': 'jpg'} - helper = (u'File name and base64 data of the file as in ' - u'x.%s;dGhpc.' - u'
It is not intented to be filled by ' - u'humans.' % exts.get(xlsf_type, 'ext')) + if xlsf_type in MEDIA_TYPES: + exts = {"audio": "mp3", "video": "avi", "photo": "jpg"} + helper = ( + "File name and base64 data of the file as in " + '' + f'x.{exts.get(xlsf_type, "ext")};dGhpc.' + "
It is not intented to be filled by " + "humans." + ) return safe_wrap(helper) - elif xlsf_type == 'barcode': - return safe_wrap('A string representing the value behind the barcode.') - elif xlsf_type == 'date': + if xlsf_type == "barcode": + return safe_wrap("A string representing the value behind the barcode.") + if xlsf_type == "date": return safe_wrap( - 'A date in the format: ' - '%s' % xlsf_date_fmt) - elif xlsf_type == 'datetime': + "A date in the format: " + f'{xlsf_date_fmt}' + ) + if xlsf_type == "datetime": return safe_wrap( - 'A datetime in the format: ' - '%s' % xlsf_datetime_fmt) - else: - return safe_wrap('?') + "A datetime in the format: " + f'{xlsf_datetime_fmt}' + ) + return safe_wrap("?") def get_autodoc_for(xform): - """ The generated documentation in a dict (HTML output) + """The generated documentation in a dict (HTML output) - line_names: example line filled with question line_names - line_values: example line filled with fake (yet valid) data - helpers: list of tuples (name, text) of helper texts. + line_names: example line filled with question line_names + line_values: example line filled with fake (yet valid) data + helpers: list of tuples (name, text) of helper texts. - Helper texts are based on type of question and accepted values """ + Helper texts are based on type of question and accepted values""" - json_survey = json.loads(xform.json) + json_survey = xform.json_dict() # setup formatting values - separator = json_survey.get('sms_separator', DEFAULT_SEPARATOR) \ - or DEFAULT_SEPARATOR + separator = json_survey.get("sms_separator", DEFAULT_SEPARATOR) or DEFAULT_SEPARATOR sms_allow_media = bool( - json_survey.get('sms_allow_media', DEFAULT_ALLOW_MEDIA) - or DEFAULT_ALLOW_MEDIA) + json_survey.get("sms_allow_media", DEFAULT_ALLOW_MEDIA) or DEFAULT_ALLOW_MEDIA + ) helpers = [] - line_names = (u'%(keyword)s' - u'%(qid)d ' % { - 'keyword': xform.sms_id_string, - 'qid': len(helpers) - }) - line_values = (u'%(keyword)s ' % { - 'keyword': xform.sms_id_string - }) - helpers.append(('keyword', u'The keyword used to identify the form.' - u'
Omit if using a form-aware URL.')) - - for group in json_survey.get('children', {}): - sms_field = group.get('sms_field', '') - if not sms_field or sms_field.lower() == 'meta': + line_names = ( + '%(keyword)s' + "%(qid)d " % {"keyword": xform.sms_id_string, "qid": len(helpers)} + ) + line_values = f'{xform.sms_id_string} ' + helpers.append( + ( + "keyword", + "The keyword used to identify the form." + "
Omit if using a form-aware URL.", + ) + ) + + for group in json_survey.get("children", {}): + sms_field = group.get("sms_field", "") + if not sms_field or sms_field.lower() == "meta": continue - line_values += (u'' - u'%(sep)s%(sms_field)s ' % { - 'sep': separator, - 'sms_field': group.get('sms_field') - }) - line_names += (u'' - u'%(sep)s%(sms_field)s ' % { - 'sep': separator, - 'sms_field': group.get('sms_field') - }) + line_values += ( + '' + "%(sep)s%(sms_field)s " + % {"sep": separator, "sms_field": group.get("sms_field")} + ) + line_names += ( + '' + "%(sep)s%(sms_field)s " + % {"sep": separator, "sms_field": group.get("sms_field")} + ) - for question in group.get('children', {}): - type_id = question.get('type') + for question in group.get("children", {}): + type_id = question.get("type") if type_id in MEDIA_TYPES and not sms_allow_media: continue @@ -213,28 +225,20 @@ def get_autodoc_for(xform): sample = get_sample_data_for(question, json_survey) if sample is None: continue - sample_name = get_sample_data_for( - question, json_survey, as_names=True) - - line_values += (u'%(value)s ' % { - 'type': type_id, - 'value': sample - }) - line_names += (u'%(value)s' - u'%(h)s ' % { - 'type': type_id, - 'value': sample_name, - 'h': qid - }) - helpers.append((sample_name, get_helper_text( - question, json_survey))) - - line_values += u'' - - return { - 'line_values': line_values, - 'line_names': line_names, - 'helpers': helpers - } + sample_name = get_sample_data_for(question, json_survey, as_names=True) + + line_values += ( + '%(value)s ' + % {"type": type_id, "value": sample} + ) + line_names += ( + '%(value)s' + "%(h)s " % {"type": type_id, "value": sample_name, "h": qid} + ) + helpers.append((sample_name, get_helper_text(question, json_survey))) + + line_values += "" + + return {"line_values": line_values, "line_names": line_names, "helpers": helpers} diff --git a/onadata/apps/sms_support/parser.py b/onadata/apps/sms_support/parser.py index a6574478a1..05ed039ee1 100644 --- a/onadata/apps/sms_support/parser.py +++ b/onadata/apps/sms_support/parser.py @@ -1,118 +1,139 @@ +# -*- coding: utf-8 -*- +# vim: ai ts=4 sts=4 et sw=4 nu +""" +SMS parser module - utility functionality to process SMS messages. +""" import base64 -import json +import binascii import logging import re -from builtins import str as text -from datetime import datetime, date +from datetime import date, datetime from io import BytesIO -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from onadata.apps.logger.models import XForm -from onadata.apps.sms_support.tools import SMS_API_ERROR, SMS_PARSING_ERROR,\ - SMS_SUBMISSION_REFUSED, sms_media_to_file, generate_instance,\ - DEFAULT_SEPARATOR, NA_VALUE, META_FIELDS, MEDIA_TYPES,\ - DEFAULT_DATE_FORMAT, DEFAULT_DATETIME_FORMAT, SMS_SUBMISSION_ACCEPTED,\ - is_last +from onadata.apps.sms_support.tools import ( + DEFAULT_DATE_FORMAT, + DEFAULT_DATETIME_FORMAT, + DEFAULT_SEPARATOR, + MEDIA_TYPES, + META_FIELDS, + NA_VALUE, + SMS_API_ERROR, + SMS_PARSING_ERROR, + SMS_SUBMISSION_ACCEPTED, + SMS_SUBMISSION_REFUSED, + generate_instance, + is_last, + sms_media_to_file, +) from onadata.libs.utils.logger_tools import dict2xform class SMSSyntaxError(ValueError): - pass + """A custom SMS syntax error exception class.""" class SMSCastingError(ValueError): + """A custom SMS type casting error exception class.""" def __init__(self, message, question=None): if question: - message = _(u"%(question)s: %(message)s") % {'question': question, - 'message': message} - super(SMSCastingError, self).__init__(message) + message = _("%(question)s: %(message)s") % { + "question": question, + "message": message, + } + super().__init__(message) -def parse_sms_text(xform, identity, text): +# pylint: disable=too-many-locals,too-many-branches,too-many-statements +def parse_sms_text(xform, identity, sms_text): # noqa C901 + """Parses an SMS text to return XForm specific answers, media, notes.""" - json_survey = json.loads(xform.json) + json_survey = xform.json_dict() - separator = json_survey.get('sms_separator', DEFAULT_SEPARATOR) \ - or DEFAULT_SEPARATOR + separator = json_survey.get("sms_separator", DEFAULT_SEPARATOR) or DEFAULT_SEPARATOR - allow_media = bool(json_survey.get('sms_allow_media', False)) + allow_media = bool(json_survey.get("sms_allow_media", False)) - xlsf_date_fmt = json_survey.get('sms_date_format', DEFAULT_DATE_FORMAT) \ - or DEFAULT_DATE_FORMAT - xlsf_datetime_fmt = json_survey.get('sms_date_format', - DEFAULT_DATETIME_FORMAT) \ + xlsf_date_fmt = ( + json_survey.get("sms_date_format", DEFAULT_DATE_FORMAT) or DEFAULT_DATE_FORMAT + ) + xlsf_datetime_fmt = ( + json_survey.get("sms_date_format", DEFAULT_DATETIME_FORMAT) or DEFAULT_DATETIME_FORMAT + ) # extract SMS data into indexed groups of values groups = {} - for group in text.split(separator)[1:]: + for group in sms_text.split(separator)[1:]: group_id, group_text = [s.strip() for s in group.split(None, 1)] groups.update({group_id: [s.strip() for s in group_text.split(None)]}) - def cast_sms_value(value, question, medias=[]): - ''' Check data type of value and return cleaned version ''' + # pylint: disable=too-many-locals,too-many-return-statements,too-many-branches + def cast_sms_value(value, question, medias=None): + """Check data type of value and return cleaned version""" - xlsf_type = question.get('type') - xlsf_name = question.get('name') - xlsf_choices = question.get('children') - xlsf_required = bool(question.get('bind', {}) - .get('required', '').lower() in ('yes', 'true')) + medias = [] if medias is None else medias + xlsf_type = question.get("type") + xlsf_name = question.get("name") + xlsf_choices = question.get("children") + xlsf_required = bool( + question.get("bind", {}).get("required", "").lower() in ("yes", "true") + ) # we don't handle constraint for now as it's a little complex and # unsafe. # xlsf_constraint=question.get('constraint') - if xlsf_required and not len(value): - raise SMSCastingError(_(u"Required field missing"), xlsf_name) + if xlsf_required and not value: + raise SMSCastingError(_("Required field missing"), xlsf_name) def safe_wrap(func): try: return func() except Exception as e: - raise SMSCastingError(_(u"%(error)s") % {'error': e}, - xlsf_name) + raise SMSCastingError(_("%(error)s") % {"error": e}, xlsf_name) from e def media_value(value, medias): - ''' handle media values + """handle media values - extract name and base64 data. - fills the media holder with (name, data) tuple ''' + extract name and base64 data. + fills the media holder with (name, data) tuple""" try: - filename, b64content = value.split(';', 1) - medias.append((filename, - base64.b64decode(b64content))) + filename, b64content = value.split(";", 1) + medias.append((filename, base64.b64decode(b64content))) return filename - except Exception as e: - raise SMSCastingError(_(u"Media file format " - u"incorrect. %(except)r") - % {'except': e}, xlsf_name) + except (AttributeError, TypeError, binascii.Error) as e: + raise SMSCastingError( + _("Media file format " "incorrect. %(except)r") % {"except": e}, + xlsf_name, + ) from e - if xlsf_type == 'text': + if xlsf_type == "text": return safe_wrap(lambda: str(value)) - elif xlsf_type == 'integer': + if xlsf_type == "integer": return safe_wrap(lambda: int(value)) - elif xlsf_type == 'decimal': + if xlsf_type == "decimal": return safe_wrap(lambda: float(value)) - elif xlsf_type == 'select one': + if xlsf_type == "select one": for choice in xlsf_choices: - if choice.get('sms_option') == value: - return choice.get('name') - raise SMSCastingError(_(u"No matching choice " - u"for '%(input)s'") - % {'input': value}, - xlsf_name) - elif xlsf_type == 'select all that apply': + if choice.get("sms_option") == value: + return choice.get("name") + raise SMSCastingError( + _("No matching choice " "for '%(input)s'") % {"input": value}, xlsf_name + ) + if xlsf_type == "select all that apply": values = [s.strip() for s in value.split()] ret_values = [] for indiv_value in values: for choice in xlsf_choices: - if choice.get('sms_option') == indiv_value: - ret_values.append(choice.get('name')) - return u" ".join(ret_values) - elif xlsf_type == 'geopoint': - err_msg = _(u"Incorrect geopoint coordinates.") + if choice.get("sms_option") == indiv_value: + ret_values.append(choice.get("name")) + return " ".join(ret_values) + if xlsf_type == "geopoint": + err_msg = _("Incorrect geopoint coordinates.") geodata = [s.strip() for s in value.split()] if len(geodata) < 2 and len(geodata) > 4: raise SMSCastingError(err_msg, xlsf_name) @@ -120,45 +141,44 @@ def media_value(value, medias): # check that latitude and longitude are floats lat, lon = [float(v) for v in geodata[:2]] # and within sphere boundaries - if lat < -90 or lat > 90 or lon < -180 and lon > 180: + if -90 > lat > 90 or -180 > lon > 180: raise SMSCastingError(err_msg, xlsf_name) if len(geodata) == 4: # check that altitude and accuracy are integers - [int(v) for v in geodata[2:4]] + for v in geodata[2:4]: + int(v) elif len(geodata) == 3: # check that altitude is integer int(geodata[2]) - except Exception as e: - raise SMSCastingError(e, xlsf_name) + except ValueError as e: + raise SMSCastingError(e, xlsf_name) from e return " ".join(geodata) - - elif xlsf_type in MEDIA_TYPES: + if xlsf_type in MEDIA_TYPES: # media content (image, video, audio) must be formatted as: # file_name;base64 encodeed content. # Example: hello.jpg;dGhpcyBpcyBteSBwaWN0dXJlIQ== return media_value(value, medias) - elif xlsf_type == 'barcode': - return safe_wrap(lambda: text(value)) - elif xlsf_type == 'date': - return safe_wrap(lambda: datetime.strptime(value, - xlsf_date_fmt).date()) - elif xlsf_type == 'datetime': - return safe_wrap(lambda: datetime.strptime(value, - xlsf_datetime_fmt)) - elif xlsf_type == 'note': - return safe_wrap(lambda: '') - raise SMSCastingError(_(u"Unsuported column '%(type)s'") - % {'type': xlsf_type}, xlsf_name) + if xlsf_type == "barcode": + return safe_wrap(lambda: str(value)) + if xlsf_type == "date": + return safe_wrap(lambda: datetime.strptime(value, xlsf_date_fmt).date()) + if xlsf_type == "datetime": + return safe_wrap(lambda: datetime.strptime(value, xlsf_datetime_fmt)) + if xlsf_type == "note": + return safe_wrap(lambda: "") + raise SMSCastingError( + _("Unsuported column '%(type)s'") % {"type": xlsf_type}, xlsf_name + ) def get_meta_value(xlsf_type, identity): - ''' XLSForm Meta field value ''' - if xlsf_type in ('deviceid', 'subscriberid', 'imei'): + """XLSForm Meta field value""" + if xlsf_type in ("deviceid", "subscriberid", "imei"): return NA_VALUE - elif xlsf_type in ('start', 'end'): + if xlsf_type in ("start", "end"): return datetime.now().isoformat() - elif xlsf_type == 'today': + if xlsf_type == "today": return date.today().isoformat() - elif xlsf_type == 'phonenumber': + if xlsf_type == "phonenumber": return identity return NA_VALUE @@ -170,24 +190,24 @@ def get_meta_value(xlsf_type, identity): notes = [] # loop on all XLSForm questions - for expected_group in json_survey.get('children', [{}]): - if not expected_group.get('type') == 'group': + for expected_group in json_survey.get("children", [{}]): + if not expected_group.get("type") == "group": # non-grouped questions are not valid for SMS continue # retrieve part of SMS text for this group - group_id = expected_group.get('sms_field') + group_id = expected_group.get("sms_field") answers = groups.get(group_id) - if not group_id or (not answers and not group_id.startswith('meta')): + if not group_id or (not answers and not group_id.startswith("meta")): # group is not meant to be filled by SMS # or hasn't been filled continue # Add a holder for this group's answers data - survey_answers.update({expected_group.get('name'): {}}) + survey_answers.update({expected_group.get("name"): {}}) # retrieve question definition for each answer - egroups = expected_group.get('children', [{}]) + egroups = expected_group.get("children", [{}]) # number of intermediate, omited questions (medias) step_back = 0 @@ -195,15 +215,15 @@ def get_meta_value(xlsf_type, identity): real_value = None - question_type = question.get('type') - if question_type in ('calculate'): + question_type = question.get("type") + if question_type == "calculate": # 'calculate' question are not implemented. # 'note' ones are just meant to be displayed on device continue - if question_type == 'note': - if not question.get('constraint', ''): - notes.append(question.get('label')) + if question_type == "note": + if not question.get("constraint", ""): + notes.append(question.get("label")) continue if not allow_media and question_type in MEDIA_TYPES: @@ -219,41 +239,44 @@ def get_meta_value(xlsf_type, identity): if question_type in META_FIELDS: # some question are not to be fed by users - real_value = get_meta_value(xlsf_type=question_type, - identity=identity) + real_value = get_meta_value(xlsf_type=question_type, identity=identity) else: # actual SMS-sent answer. # Only last answer/question of each group is allowed # to have multiple spaces if is_last(idx, egroups): - answer = u" ".join(answers[idx:]) + answer = " ".join(answers[idx:]) else: answer = answers[sidx] if real_value is None: # retrieve actual value and fail if it doesn't meet reqs. - real_value = cast_sms_value(answer, - question=question, medias=medias) + real_value = cast_sms_value(answer, question=question, medias=medias) # set value to its question name - survey_answers[expected_group.get('name')] \ - .update({question.get('name'): real_value}) + survey_answers[expected_group.get("name")].update( + {question.get("name"): real_value} + ) return survey_answers, medias, notes -def process_incoming_smses(username, incomings, - id_string=None): - ''' Process Incoming (identity, text[, id_string]) SMS ''' +# pylint: disable=too-many-statements +def process_incoming_smses(username, incomings, id_string=None): # noqa C901 + """Process Incoming (identity, text[, id_string]) SMS""" xforms = [] medias = [] xforms_notes = [] responses = [] json_submissions = [] - resp_str = {'success': _(u"[SUCCESS] Your submission has been accepted. " - u"It's ID is {{ id }}.")} + resp_str = { + "success": _( + "[SUCCESS] Your submission has been accepted. " "It's ID is {{ id }}." + ) + } + # pylint: disable=too-many-branches def process_incoming(incoming, id_string): # assign variables if len(incoming) >= 2: @@ -263,86 +286,102 @@ def process_incoming(incoming, id_string): if id_string is None and len(incoming) >= 3: id_string = incoming[2] else: - responses.append({'code': SMS_API_ERROR, - 'text': _(u"Missing 'identity' " - u"or 'text' field.")}) + responses.append( + { + "code": SMS_API_ERROR, + "text": _("Missing 'identity' " "or 'text' field."), + } + ) return - if not len(identity.strip()) or not len(text.strip()): - responses.append({'code': SMS_API_ERROR, - 'text': _(u"'identity' and 'text' fields can " - u"not be empty.")}) + if not identity.strip() or not text.strip(): + responses.append( + { + "code": SMS_API_ERROR, + "text": _("'identity' and 'text' fields can " "not be empty."), + } + ) return # if no id_string has been supplied # we expect the SMS to be prefixed with the form's sms_id_string if id_string is None: keyword, text = [s.strip() for s in text.split(None, 1)] - xform = XForm.objects.get(user__username=username, - sms_id_string=keyword) + xform = XForm.objects.get(user__username=username, sms_id_string=keyword) else: - xform = XForm.objects.get(user__username=username, - id_string=id_string) + xform = XForm.objects.get(user__username=username, id_string=id_string) if not xform.allows_sms: - responses.append({'code': SMS_SUBMISSION_REFUSED, - 'text': _(u"The form '%(id_string)s' does not " - u"accept SMS submissions.") - % {'id_string': xform.id_string}}) + responses.append( + { + "code": SMS_SUBMISSION_REFUSED, + "text": _( + "The form '%(id_string)s' does not " "accept SMS submissions." + ) + % {"id_string": xform.id_string}, + } + ) return # parse text into a dict object of groups with values - json_submission, medias_submission, notes = parse_sms_text(xform, - identity, - text) + json_submission, medias_submission, notes = parse_sms_text( + xform, identity, text + ) # retrieve sms_response if exist in the form. - json_survey = json.loads(xform.json) - if json_survey.get('sms_response'): - resp_str.update({'success': json_survey.get('sms_response')}) + json_survey = xform.json_dict() + if json_survey.get("sms_response"): + resp_str.update({"success": json_survey.get("sms_response")}) # check that the form contains at least one filled group - meta_groups = sum([1 for k in list(json_submission) - if k.startswith('meta')]) + meta_groups = sum([1 for k in list(json_submission) if k.startswith("meta")]) if len(list(json_submission)) <= meta_groups: - responses.append({'code': SMS_PARSING_ERROR, - 'text': _(u"There must be at least one group of " - u"questions filled.")}) + responses.append( + { + "code": SMS_PARSING_ERROR, + "text": _( + "There must be at least one group of " "questions filled." + ), + } + ) return # check that required fields have been filled - required_fields = [f.get('name') - for g in json_survey.get('children', {}) - for f in g.get('children', {}) - if f.get('bind', {}).get('required', 'no') == 'yes'] + required_fields = [ + f.get("name") + for g in json_survey.get("children", {}) + for f in g.get("children", {}) + if f.get("bind", {}).get("required", "no") == "yes" + ] submitted_fields = {} for group in json_submission.values(): submitted_fields.update(group) for field in required_fields: if not submitted_fields.get(field): - responses.append({'code': SMS_SUBMISSION_REFUSED, - 'text': _(u"Required field `%(field)s` is " - u"missing.") % {'field': field}}) + responses.append( + { + "code": SMS_SUBMISSION_REFUSED, + "text": _(f"Required field `{field}` is " "missing."), + } + ) return # convert dict object into an XForm string - xml_submission = dict2xform(jsform=json_submission, - form_id=xform.id_string) + xml_submission = dict2xform(jsform=json_submission, form_id=xform.id_string) # compute notes data = {} - for g in json_submission.values(): - data.update(g) + for group in json_submission.values(): + data.update(group) for idx, note in enumerate(notes): try: - notes[idx] = note.replace('${', '{').format(**data) - except Exception as e: - logging.exception(_(u'Updating note threw exception: %s' - % text(e))) + notes[idx] = note.replace("${", "{").format(**data) + except AttributeError as e: + logging.exception("Updating note threw exception: %s", str(e)) # process_incoming expectes submission to be a file-like object - xforms.append(BytesIO(xml_submission.encode('utf-8'))) + xforms.append(BytesIO(xml_submission.encode("utf-8"))) medias.append(medias_submission) json_submissions.append(json_submission) xforms_notes.append(notes) @@ -350,30 +389,32 @@ def process_incoming(incoming, id_string): for incoming in incomings: try: process_incoming(incoming, id_string) - except Exception as e: - responses.append({'code': SMS_PARSING_ERROR, 'text': text(e)}) + except (SMSCastingError, SMSSyntaxError, ValueError) as e: + responses.append({"code": SMS_PARSING_ERROR, "text": str(e)}) for idx, xform in enumerate(xforms): # generate_instance expects media as a request.FILES.values() list xform_medias = [sms_media_to_file(f, n) for n, f in medias[idx]] # create the instance in the data base - response = generate_instance(username=username, - xml_file=xform, - media_files=xform_medias) - if response.get('code') == SMS_SUBMISSION_ACCEPTED: - success_response = re.sub(r'{{\s*[i,d,I,D]{2}\s*}}', - response.get('id'), - resp_str.get('success'), re.I) + response = generate_instance( + username=username, xml_file=xform, media_files=xform_medias + ) + if response.get("code") == SMS_SUBMISSION_ACCEPTED: + success_response = re.sub( + r"{{\s*[i,d,I,D]{2}\s*}}", + response.get("id"), + resp_str.get("success"), + re.I, + ) # extend success_response with data from the answers data = {} - for g in json_submissions[idx].values(): - data.update(g) - success_response = success_response.replace('${', - '{').format(**data) - response.update({'text': success_response}) + for group in json_submissions[idx].values(): + data.update(group) + success_response = success_response.replace("${", "{").format(**data) + response.update({"text": success_response}) # add sendouts (notes) - response.update({'sendouts': xforms_notes[idx]}) + response.update({"sendouts": xforms_notes[idx]}) responses.append(response) return responses diff --git a/onadata/apps/sms_support/providers/__init__.py b/onadata/apps/sms_support/providers/__init__.py index fa353ade61..09b01f9549 100644 --- a/onadata/apps/sms_support/providers/__init__.py +++ b/onadata/apps/sms_support/providers/__init__.py @@ -1,82 +1,106 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu +""" +sms_support.providers +""" from __future__ import absolute_import from django.http import HttpResponse from django.views.decorators.csrf import csrf_exempt -from onadata.apps.sms_support.providers.smssync import \ - autodoc as autodoc_smssync -from onadata.apps.sms_support.providers.smssync import \ - import_submission as imp_sub_smssync -from onadata.apps.sms_support.providers.smssync import \ - import_submission_for_form as imp_sub_form_smssync -from onadata.apps.sms_support.providers.telerivet import \ - autodoc as autodoc_telerivet -from onadata.apps.sms_support.providers.telerivet import \ - import_submission as imp_sub_telerivet -from onadata.apps.sms_support.providers.telerivet import \ - import_submission_for_form as imp_sub_form_telerivet +from onadata.apps.sms_support.providers.smssync import autodoc as autodoc_smssync +from onadata.apps.sms_support.providers.smssync import ( + import_submission as imp_sub_smssync, +) +from onadata.apps.sms_support.providers.smssync import ( + import_submission_for_form as imp_sub_form_smssync, +) +from onadata.apps.sms_support.providers.telerivet import autodoc as autodoc_telerivet +from onadata.apps.sms_support.providers.telerivet import ( + import_submission as imp_sub_telerivet, +) +from onadata.apps.sms_support.providers.telerivet import ( + import_submission_for_form as imp_sub_form_telerivet, +) from onadata.apps.sms_support.providers.textit import autodoc as autodoc_textit -from onadata.apps.sms_support.providers.textit import \ - import_submission as imp_sub_textit -from onadata.apps.sms_support.providers.textit import \ - import_submission_for_form as imp_sub_form_textit +from onadata.apps.sms_support.providers.textit import ( + import_submission as imp_sub_textit, +) +from onadata.apps.sms_support.providers.textit import ( + import_submission_for_form as imp_sub_form_textit, +) from onadata.apps.sms_support.providers.twilio import autodoc as autodoc_twilio -from onadata.apps.sms_support.providers.twilio import \ - import_submission as imp_sub_twilio -from onadata.apps.sms_support.providers.twilio import \ - import_submission_for_form as imp_sub_form_twilio +from onadata.apps.sms_support.providers.twilio import ( + import_submission as imp_sub_twilio, +) +from onadata.apps.sms_support.providers.twilio import ( + import_submission_for_form as imp_sub_form_twilio, +) -SMSSYNC = 'smssync' -TELERIVET = 'telerivet' -TWILIO = 'twilio' -TEXTIT = 'textit' +SMSSYNC = "smssync" +TELERIVET = "telerivet" +TWILIO = "twilio" +TEXTIT = "textit" PROVIDERS = { - SMSSYNC: {'name': u"SMS Sync", - 'imp': imp_sub_smssync, - 'imp_form': imp_sub_form_smssync, - 'doc': autodoc_smssync}, - TELERIVET: {'name': u"Telerivet", - 'imp': imp_sub_telerivet, - 'imp_form': imp_sub_form_telerivet, - 'doc': autodoc_telerivet}, - TWILIO: {'name': u"Twilio", - 'imp': imp_sub_twilio, - 'imp_form': imp_sub_form_twilio, - 'doc': autodoc_twilio}, - TEXTIT: {'name': u"Text It", - 'imp': imp_sub_textit, - 'imp_form': imp_sub_form_textit, - 'doc': autodoc_textit} + SMSSYNC: { + "name": "SMS Sync", + "imp": imp_sub_smssync, + "imp_form": imp_sub_form_smssync, + "doc": autodoc_smssync, + }, + TELERIVET: { + "name": "Telerivet", + "imp": imp_sub_telerivet, + "imp_form": imp_sub_form_telerivet, + "doc": autodoc_telerivet, + }, + TWILIO: { + "name": "Twilio", + "imp": imp_sub_twilio, + "imp_form": imp_sub_form_twilio, + "doc": autodoc_twilio, + }, + TEXTIT: { + "name": "Text It", + "imp": imp_sub_textit, + "imp_form": imp_sub_form_textit, + "doc": autodoc_textit, + }, } +# pylint: disable=unused-argument def unknown_service(request, username=None, id_string=None): - """ 400 view for request with unknown service name """ - r = HttpResponse(u"Unknown SMS Gateway Service", content_type='text/plain') - r.status_code = 400 - return r + """400 view for request with unknown service name""" + response = HttpResponse("Unknown SMS Gateway Service", content_type="text/plain") + response.status_code = 400 + return response @csrf_exempt def import_submission(request, username, service): - """ Proxy to the service's import_submission view """ - return PROVIDERS.get(service.lower(), {}).get( - 'imp', unknown_service)(request, username) + """Proxy to the service's import_submission view""" + return PROVIDERS.get(service.lower(), {}).get("imp", unknown_service)( + request, username + ) @csrf_exempt def import_submission_for_form(request, username, id_string, service): - """ Proxy to the service's import_submission_for_form view """ - return PROVIDERS.get(service.lower(), {}).get( - 'imp_form', unknown_service)(request, username, id_string) + """Proxy to the service's import_submission_for_form view""" + return PROVIDERS.get(service.lower(), {}).get("imp_form", unknown_service)( + request, username, id_string + ) def providers_doc(url_root, username, id_string): - return [{'id': pid, - 'name': p.get('name'), - 'doc': p.get('doc')(url_root, username, id_string)} - for pid, p in PROVIDERS.items()] + return [ + { + "id": pid, + "name": p.get("name"), + "doc": p.get("doc")(url_root, username, id_string), + } + for pid, p in PROVIDERS.items() + ] diff --git a/onadata/apps/sms_support/providers/smssync.py b/onadata/apps/sms_support/providers/smssync.py index e5159bc000..b146c90b04 100644 --- a/onadata/apps/sms_support/providers/smssync.py +++ b/onadata/apps/sms_support/providers/smssync.py @@ -7,117 +7,128 @@ See: http://smssync.ushahidi.com/doc """ -import json import datetime -from django.http import HttpResponse +from django.http import JsonResponse from django.urls import reverse -from django.utils.translation import ugettext as _ -from django.views.decorators.http import require_POST +from django.utils.translation import gettext as _ from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_POST -from onadata.apps.sms_support.tools import SMS_API_ERROR,\ - SMS_SUBMISSION_ACCEPTED from onadata.apps.sms_support.parser import process_incoming_smses +from onadata.apps.sms_support.tools import SMS_API_ERROR, SMS_SUBMISSION_ACCEPTED def autodoc(url_root, username, id_string): - urla = url_root + reverse('sms_submission_api', - kwargs={'username': username, - 'service': 'smssync'}) - urlb = url_root + reverse('sms_submission_form_api', - kwargs={'username': username, - 'id_string': id_string, - 'service': 'smssync'}) - doc = (u'

' + - _(u"%(service)s Instructions:") - % {'service': u'' - u'Ushaidi\'s SMS Sync'} + - u'

  1. ' + - _(u"Download the SMS Sync App on your phone serving " - u"as a gateway.") + '
  2. ' + - _(u"Configure the app to point to one of the following URLs") + - u'
    %(urla)s' + - u'
    %(urlb)s
    ' + - _(u"Optionnaly set a keyword to prevent non-formhub " - u"messages to be sent.") + - '
  3. ' + - _(u"In the preferences, tick the box to allow " - u"replies from the server.") + - '

' + - _(u"That's it. Now Send an SMS Formhub submission to the number " - u"of that phone. It will create a submission on Formhub.") + - u'

') % {'urla': urla, 'urlb': urlb} + """Returns SMSSync integration documentation.""" + urla = url_root + reverse( + "sms_submission_api", kwargs={"username": username, "service": "smssync"} + ) + urlb = url_root + reverse( + "sms_submission_form_api", + kwargs={"username": username, "id_string": id_string, "service": "smssync"}, + ) + doc = ( + "

" + + _("%(service)s Instructions:") + % { + "service": '' + "Ushaidi's SMS Sync" + } + + "

  1. " + + _("Download the SMS Sync App on your phone serving " "as a gateway.") + + "
  2. " + + _("Configure the app to point to one of the following URLs") + + '
    %(urla)s' + + "
    %(urlb)s
    " + + _("Optionnaly set a keyword to prevent non-formhub " "messages to be sent.") + + "
  3. " + + _("In the preferences, tick the box to allow " "replies from the server.") + + "

" + + _( + "That's it. Now Send an SMS Formhub submission to the number " + "of that phone. It will create a submission on Formhub." + ) + + "

" + ) % {"urla": urla, "urlb": urlb} return doc def get_response(data): - message = data.get('text') - if data.get('code') == SMS_API_ERROR: + """Return a JSON formatted HttpResponse based on the ``data`` provided.""" + message = data.get("text") + if data.get("code") == SMS_API_ERROR: success = False message = None - elif data.get('code') != SMS_SUBMISSION_ACCEPTED: + elif data.get("code") != SMS_SUBMISSION_ACCEPTED: success = True - message = _(u"[ERROR] %s") % message + message = _("[ERROR] %s") % message else: success = True - response = { - "payload": { - "success": success, - "task": "send"}} + response = {"payload": {"success": success, "task": "send"}} if message: - messages = [{"to": data.get('identity'), "message": message}] - sendouts = data.get('sendouts', []) + messages = [{"to": data.get("identity"), "message": message}] + sendouts = data.get("sendouts", []) if len(sendouts): - messages += [{"to": data.get('identity'), "message": text} - for text in sendouts] - response['payload'].update({"messages": messages}) - return HttpResponse(json.dumps(response), content_type='application/json') + messages += [ + {"to": data.get("identity"), "message": text} for text in sendouts + ] + response["payload"].update({"messages": messages}) + + return JsonResponse(response) @require_POST @csrf_exempt def import_submission(request, username): - """ Proxy to import_submission_for_form with None as id_string """ + """Proxy to import_submission_for_form with None as id_string""" return import_submission_for_form(request, username, None) @require_POST @csrf_exempt def import_submission_for_form(request, username, id_string): - """ Retrieve and process submission from SMSSync Request """ + """Retrieve and process submission from SMSSync Request""" - sms_identity = request.POST.get('from', '').strip() - sms_text = request.POST.get('message', '').strip() - now_timestamp = datetime.datetime.now().strftime('%s') - sent_timestamp = request.POST.get('sent_timestamp', now_timestamp).strip() + sms_identity = request.POST.get("from", "").strip() + sms_text = request.POST.get("message", "").strip() + now_timestamp = datetime.datetime.now().strftime("%s") + sent_timestamp = request.POST.get("sent_timestamp", now_timestamp).strip() try: sms_time = datetime.datetime.fromtimestamp(float(sent_timestamp)) except ValueError: sms_time = datetime.datetime.now() - return process_message_for_smssync(username=username, - sms_identity=sms_identity, - sms_text=sms_text, - sms_time=sms_time, - id_string=id_string) + return process_message_for_smssync( + username=username, + sms_identity=sms_identity, + sms_text=sms_text, + sms_time=sms_time, + id_string=id_string, + ) -def process_message_for_smssync(username, - sms_identity, sms_text, sms_time, id_string): - """ Process a text instance and return in SMSSync expected format """ +# pylint: disable=unused-argument +def process_message_for_smssync(username, sms_identity, sms_text, sms_time, id_string): + """Process a text instance and return in SMSSync expected format""" if not sms_identity or not sms_text: - return get_response({'code': SMS_API_ERROR, - 'text': _(u"`identity` and `message` are " - u"both required and must not be " - u"empty.")}) + return get_response( + { + "code": SMS_API_ERROR, + "text": _( + "`identity` and `message` are " + "both required and must not be " + "empty." + ), + } + ) incomings = [(sms_identity, sms_text)] response = process_incoming_smses(username, incomings, id_string)[-1] - response.update({'identity': sms_identity}) + response.update({"identity": sms_identity}) return get_response(response) diff --git a/onadata/apps/sms_support/providers/telerivet.py b/onadata/apps/sms_support/providers/telerivet.py index ae9f37987a..69213ca16c 100644 --- a/onadata/apps/sms_support/providers/telerivet.py +++ b/onadata/apps/sms_support/providers/telerivet.py @@ -4,70 +4,74 @@ See: http://telerivet.com/help/api/webhook/receiving """ -import json import datetime -from django.http import HttpResponse +from django.http import JsonResponse from django.urls import reverse -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.views.decorators.http import require_POST from django.views.decorators.csrf import csrf_exempt -from onadata.apps.sms_support.tools import SMS_API_ERROR,\ - SMS_SUBMISSION_ACCEPTED +from onadata.apps.sms_support.tools import SMS_API_ERROR, SMS_SUBMISSION_ACCEPTED from onadata.apps.sms_support.parser import process_incoming_smses def autodoc(url_root, username, id_string): - urla = url_root + reverse('sms_submission_api', - kwargs={'username': username, - 'service': 'telerivet'}) - urlb = url_root + reverse('sms_submission_form_api', - kwargs={'username': username, - 'id_string': id_string, - 'service': 'telerivet'}) - doc = (u'

' + - _(u"%(service)s Instructions:") - % {'service': u'' - u'Telerivet\'s Webhook API'} + - u'

  1. ' + - _(u"Sign in to Telerivet.com and go to Service Page.") + - u'
  2. ' + - _(u"Follow instructions to add an application with either URL:") + - u'
    %(urla)s' + - u'
    %(urlb)s
    ' + - u'

' + - _(u"That's it. Now Send an SMS Formhub submission to your Telerivet" - u" phone number. It will create a submission on Formhub.") + - u'

') % {'urla': urla, 'urlb': urlb} + """Returns Telerivet integration documentation.""" + urla = url_root + reverse( + "sms_submission_api", kwargs={"username": username, "service": "telerivet"} + ) + urlb = url_root + reverse( + "sms_submission_form_api", + kwargs={"username": username, "id_string": id_string, "service": "telerivet"}, + ) + doc = ( + "

" + + _("%(service)s Instructions:") + % {"service": '' "Telerivet's Webhook API"} + + "

  1. " + + _("Sign in to Telerivet.com and go to Service Page.") + + "
  2. " + + _("Follow instructions to add an application with either URL:") + + '
    %(urla)s' + + "
    %(urlb)s
    " + + "

" + + _( + "That's it. Now Send an SMS Formhub submission to your Telerivet" + " phone number. It will create a submission on Formhub." + ) + + "

" + ) % {"urla": urla, "urlb": urlb} return doc def get_response(data): + """Return a JSON formatted HttpResponse based on the ``data`` provided.""" - message = data.get('text') + message = data.get("text") - if data.get('code') == SMS_API_ERROR: + if data.get("code") == SMS_API_ERROR: message = None - elif data.get('code') != SMS_SUBMISSION_ACCEPTED: - message = _(u"[ERROR] %s") % message + elif data.get("code") != SMS_SUBMISSION_ACCEPTED: + message = _(f"[ERROR] {message}") response = {} if message: messages = [{"content": message}] - sendouts = data.get('sendouts', []) + sendouts = data.get("sendouts", []) if len(sendouts): messages += [{"content": text} for text in sendouts] response.update({"messages": messages}) - return HttpResponse(json.dumps(response), content_type='application/json') + + return JsonResponse(response) @require_POST @csrf_exempt def import_submission(request, username): - """ Proxy to import_submission_for_form with None as id_string """ + """Proxy to import_submission_for_form with None as id_string""" return import_submission_for_form(request, username, None) @@ -75,33 +79,43 @@ def import_submission(request, username): @require_POST @csrf_exempt def import_submission_for_form(request, username, id_string): - """ Retrieve and process submission from SMSSync Request """ + """Retrieve and process submission from SMSSync Request""" - sms_identity = request.POST.get('from_number', '').strip() - sms_text = request.POST.get('content', '').strip() - now_timestamp = datetime.datetime.now().strftime('%s') - sent_timestamp = request.POST.get('time_created', now_timestamp).strip() + sms_identity = request.POST.get("from_number", "").strip() + sms_text = request.POST.get("content", "").strip() + now_timestamp = datetime.datetime.now().strftime("%s") + sent_timestamp = request.POST.get("time_created", now_timestamp).strip() try: sms_time = datetime.datetime.fromtimestamp(float(sent_timestamp)) except ValueError: sms_time = datetime.datetime.now() - return process_message_for_telerivet(username=username, - sms_identity=sms_identity, - sms_text=sms_text, - sms_time=sms_time, - id_string=id_string) + return process_message_for_telerivet( + username=username, + sms_identity=sms_identity, + sms_text=sms_text, + sms_time=sms_time, + id_string=id_string, + ) -def process_message_for_telerivet(username, - sms_identity, sms_text, sms_time, id_string): - """ Process a text instance and return in SMSSync expected format """ +# pylint: disable=unused-argument +def process_message_for_telerivet( + username, sms_identity, sms_text, sms_time, id_string +): + """Process a text instance and return in SMSSync expected format""" if not sms_identity or not sms_text: - return get_response({'code': SMS_API_ERROR, - 'text': _(u"`identity` and `message` are " - u"both required and must not be " - u"empty.")}) + return get_response( + { + "code": SMS_API_ERROR, + "text": _( + "`identity` and `message` are " + "both required and must not be " + "empty." + ), + } + ) incomings = [(sms_identity, sms_text)] response = process_incoming_smses(username, incomings, id_string)[-1] diff --git a/onadata/apps/sms_support/providers/textit.py b/onadata/apps/sms_support/providers/textit.py index 03fa01ac1a..3cb42443b4 100644 --- a/onadata/apps/sms_support/providers/textit.py +++ b/onadata/apps/sms_support/providers/textit.py @@ -8,127 +8,143 @@ See: https://textit.in/api/v1/webhook/ """ import datetime -import dateutil -import json -from django.http import HttpResponse +from django.http import HttpResponse, JsonResponse from django.urls import reverse -from django.utils.translation import ugettext as _ -from django.views.decorators.http import require_POST +from django.utils.translation import gettext as _ from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_POST + +import dateutil -from onadata.apps.sms_support.tools import SMS_API_ERROR,\ - SMS_SUBMISSION_ACCEPTED from onadata.apps.sms_support.parser import process_incoming_smses +from onadata.apps.sms_support.tools import SMS_API_ERROR, SMS_SUBMISSION_ACCEPTED -TEXTIT_URL = 'https://api.textit.in/api/v1/sms.json' +TEXTIT_URL = "https://api.textit.in/api/v1/sms.json" def autodoc(url_root, username, id_string): - urla = url_root + reverse('sms_submission_api', - kwargs={'username': username, - 'service': 'textit'}) - urlb = url_root + reverse('sms_submission_form_api', - kwargs={'username': username, - 'id_string': id_string, - 'service': 'textit'}) - doc = (u'

' + - _(u"%(service)s Instructions:") - % {'service': u'' - u'TextIt\'s Webhook API'} + - u'

  1. ' + - _(u"Sign in to TextIt.in and go to Account Page.") + - u'
  2. ' + - _(u"Tick “Incoming SMS Messages” and set Webhook URL to either:") + - u'
    %(urla)s' + - u'
    %(urlb)s
    ' + - u'

' + - _(u"That's it. Now Send an SMS Formhub submission to your TextIt" - u" phone number. It will create a submission on Formhub.") + - u'

') % {'urla': urla, 'urlb': urlb} + """Returns the documentation string for the provider.""" + urla = url_root + reverse( + "sms_submission_api", kwargs={"username": username, "service": "textit"} + ) + urlb = url_root + reverse( + "sms_submission_form_api", + kwargs={"username": username, "id_string": id_string, "service": "textit"}, + ) + doc = ( + "

" + + _("%(service)s Instructions:") + % {"service": '' "TextIt's Webhook API"} + + "

  1. " + + _("Sign in to TextIt.in and go to Account Page.") + + "
  2. " + + _("Tick “Incoming SMS Messages” and set Webhook URL to either:") + + '
    %(urla)s' + + "
    %(urlb)s
    " + + "

" + + _( + "That's it. Now Send an SMS Formhub submission to your TextIt" + " phone number. It will create a submission on Formhub." + ) + + "

" + ) % {"urla": urla, "urlb": urlb} return doc def get_response(data): + """Sends SMS ``data`` to textit via send_sms_via_textit() function.""" - message = data.get('text') - if data.get('code') == SMS_API_ERROR: + message = data.get("text") + if data.get("code") == SMS_API_ERROR: message = None - elif data.get('code') != SMS_SUBMISSION_ACCEPTED: - message = _(u"[ERROR] %s") % message + elif data.get("code") != SMS_SUBMISSION_ACCEPTED: + message = _("[ERROR] %s") % message # send a response if message: - messages = [message, ] - sendouts = data.get('sendouts', []) + messages = [ + message, + ] + sendouts = data.get("sendouts", []) if len(sendouts): messages += sendouts for text in messages: - payload = data.get('payload', {}) - payload.update({'text': text}) - if payload.get('phone'): + payload = data.get("payload", {}) + payload.update({"text": text}) + if payload.get("phone"): send_sms_via_textit(payload) return HttpResponse() def send_sms_via_textit(payload): - response = {"phone": [payload.get('phone')], - "text": payload.get('text')} + """Returns the JsonResponse of the SMS JSOn payload for text it.""" + response = {"phone": [payload.get("phone")], "text": payload.get("text")} - return HttpResponse(json.dumps(response), content_type='application/json') + return JsonResponse(response) @require_POST @csrf_exempt def import_submission(request, username): - """ Proxy to import_submission_for_form with None as id_string """ + """Proxy to import_submission_for_form with None as id_string""" return import_submission_for_form(request, username, None) @require_POST @csrf_exempt def import_submission_for_form(request, username, id_string): - """ Retrieve and process submission from SMSSync Request """ + """Retrieve and process submission from SMSSync Request""" - sms_event = request.POST.get('event', '').strip() + sms_event = request.POST.get("event", "").strip() - if not sms_event == 'mo_sms': + if not sms_event == "mo_sms": return HttpResponse() - sms_identity = request.POST.get('phone', '').strip() - sms_relayer = request.POST.get('relayer', '').strip() - sms_text = request.POST.get('text', '').strip() + sms_identity = request.POST.get("phone", "").strip() + sms_relayer = request.POST.get("relayer", "").strip() + sms_text = request.POST.get("text", "").strip() now_time = datetime.datetime.now().isoformat() - sent_time = request.POST.get('time', now_time).strip() + sent_time = request.POST.get("time", now_time).strip() try: sms_time = dateutil.parser.parse(sent_time) except ValueError: sms_time = datetime.datetime.now() - return process_message_for_textit(username=username, - sms_identity=sms_identity, - sms_text=sms_text, - sms_time=sms_time, - id_string=id_string, - payload={'phone': sms_identity, - 'relayer': sms_relayer}) + return process_message_for_textit( + username=username, + sms_identity=sms_identity, + sms_text=sms_text, + sms_time=sms_time, + id_string=id_string, + payload={"phone": sms_identity, "relayer": sms_relayer}, + ) -def process_message_for_textit(username, sms_identity, sms_text, sms_time, - id_string, payload={}): - """ Process a text instance and return in SMSSync expected format """ +# pylint: disable=unused-argument,too-many-arguments +def process_message_for_textit( + username, sms_identity, sms_text, sms_time, id_string, payload=None +): + """Process a text instance and return in SMSSync expected format""" + payload = {} if payload is None else payload if not sms_identity or not sms_text: - return get_response({'code': SMS_API_ERROR, - 'text': _(u"`identity` and `message` are " - u"both required and must not be " - u"empty.")}) + return get_response( + { + "code": SMS_API_ERROR, + "text": _( + "`identity` and `message` are " + "both required and must not be " + "empty." + ), + } + ) incomings = [(sms_identity, sms_text)] response = process_incoming_smses(username, incomings, id_string)[-1] - response.update({'payload': payload}) + response.update({"payload": payload}) return get_response(response) diff --git a/onadata/apps/sms_support/providers/twilio.py b/onadata/apps/sms_support/providers/twilio.py index c9fd62e815..d2eeb2b2c7 100644 --- a/onadata/apps/sms_support/providers/twilio.py +++ b/onadata/apps/sms_support/providers/twilio.py @@ -11,71 +11,80 @@ import datetime -from dict2xml import dict2xml from django.http import HttpResponse from django.urls import reverse -from django.utils.translation import ugettext as _ -from django.views.decorators.http import require_POST +from django.utils.translation import gettext as _ from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_POST + +from dict2xml import dict2xml -from onadata.apps.sms_support.tools import SMS_API_ERROR,\ - SMS_SUBMISSION_ACCEPTED from onadata.apps.sms_support.parser import process_incoming_smses +from onadata.apps.sms_support.tools import SMS_API_ERROR, SMS_SUBMISSION_ACCEPTED def autodoc(url_root, username, id_string): - urla = url_root + reverse('sms_submission_api', - kwargs={'username': username, - 'service': 'twilio'}) - urlb = url_root + reverse('sms_submission_form_api', - kwargs={'username': username, - 'id_string': id_string, - 'service': 'twilio'}) - doc = (u'

' + - _(u"%(service)s Instructions:") - % {'service': u'' - u'Twilio\'s SMS Request'} + - u'

  1. ' + - _(u"Sign in to Twilio.com and go your Application.") + - u'
  2. ' + - _(u"Follow instructions to add one of the following URLs, " - u"selecting the HTTP POST method:") + - u'
    %(urla)s' + - u'
    %(urlb)s
    ' + - u'

' + - _(u"That's it. Now Send an SMS Formhub submission to your Twilio" - u" phone number. It will create a submission on Formhub.") + - u'

') % {'urla': urla, 'urlb': urlb} + """Returns Twilio integration documentation.""" + urla = url_root + reverse( + "sms_submission_api", kwargs={"username": username, "service": "twilio"} + ) + urlb = url_root + reverse( + "sms_submission_form_api", + kwargs={"username": username, "id_string": id_string, "service": "twilio"}, + ) + doc = ( + "

" + + _("%(service)s Instructions:") + % {"service": '' "Twilio's SMS Request"} + + "

  1. " + + _("Sign in to Twilio.com and go your Application.") + + "
  2. " + + _( + "Follow instructions to add one of the following URLs, " + "selecting the HTTP POST method:" + ) + + '
    %(urla)s' + + "
    %(urlb)s
    " + + "

" + + _( + "That's it. Now Send an SMS Formhub submission to your Twilio" + " phone number. It will create a submission on Formhub." + ) + + "

" + ) % {"urla": urla, "urlb": urlb} return doc def get_response(data): + """Return an XML formatted HttpResponse based on the ``data`` provided.""" - xml_head = u'' - response_dict = {'Response': {}} - message = data.get('text') + xml_head = '' + response_dict = {"Response": {}} + message = data.get("text") - if data.get('code') == SMS_API_ERROR: + if data.get("code") == SMS_API_ERROR: message = None - elif data.get('code') != SMS_SUBMISSION_ACCEPTED: - message = _(u"[ERROR] %s") % message + elif data.get("code") != SMS_SUBMISSION_ACCEPTED: + message = _("[ERROR] %s") % message if message: - messages = [message, ] - sendouts = data.get('sendouts', []) + messages = [ + message, + ] + sendouts = data.get("sendouts", []) if len(sendouts): messages += sendouts - response_dict.update({"Response": {'Sms': messages}}) + response_dict.update({"Response": {"Sms": messages}}) response = xml_head + dict2xml(response_dict) - return HttpResponse(response, content_type='text/xml') + return HttpResponse(response, content_type="text/xml") @require_POST @csrf_exempt def import_submission(request, username): - """ Proxy to import_submission_for_form with None as id_string """ + """Proxy to import_submission_for_form with None as id_string""" return import_submission_for_form(request, username, None) @@ -83,33 +92,41 @@ def import_submission(request, username): @require_POST @csrf_exempt def import_submission_for_form(request, username, id_string): - """ Retrieve and process submission from SMSSync Request """ + """Retrieve and process submission from SMSSync Request""" - sms_identity = request.POST.get('From', '').strip() - sms_text = request.POST.get('Body', '').strip() - now_timestamp = datetime.datetime.now().strftime('%s') - sent_timestamp = request.POST.get('time_created', now_timestamp).strip() + sms_identity = request.POST.get("From", "").strip() + sms_text = request.POST.get("Body", "").strip() + now_timestamp = datetime.datetime.now().strftime("%s") + sent_timestamp = request.POST.get("time_created", now_timestamp).strip() try: sms_time = datetime.datetime.fromtimestamp(float(sent_timestamp)) except ValueError: sms_time = datetime.datetime.now() - return process_message_for_twilio(username=username, - sms_identity=sms_identity, - sms_text=sms_text, - sms_time=sms_time, - id_string=id_string) + return process_message_for_twilio( + username=username, + sms_identity=sms_identity, + sms_text=sms_text, + sms_time=sms_time, + id_string=id_string, + ) -def process_message_for_twilio(username, - sms_identity, sms_text, sms_time, id_string): - """ Process a text instance and return in SMSSync expected format """ +# pylint: disable=unused-argument +def process_message_for_twilio(username, sms_identity, sms_text, sms_time, id_string): + """Process a text instance and return in SMSSync expected format""" if not sms_identity or not sms_text: - return get_response({'code': SMS_API_ERROR, - 'text': _(u"`identity` and `message` are " - u"both required and must not be " - u"empty.")}) + return get_response( + { + "code": SMS_API_ERROR, + "text": _( + "`identity` and `message` are " + "both required and must not be " + "empty." + ), + } + ) incomings = [(sms_identity, sms_text)] response = process_incoming_smses(username, incomings, id_string)[-1] diff --git a/onadata/apps/sms_support/tests/test_base_sms.py b/onadata/apps/sms_support/tests/test_base_sms.py index 2cca175e5d..eb4dcd53f5 100644 --- a/onadata/apps/sms_support/tests/test_base_sms.py +++ b/onadata/apps/sms_support/tests/test_base_sms.py @@ -1,49 +1,61 @@ +# -*- coding: utf-8 -*- +""" +TestBaseSMS - base class for sms_support test cases. +""" import os import string import random -from builtins import range from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.models import XForm from onadata.apps.sms_support.parser import process_incoming_smses +def random_identity(): + """Returns some random digits and ascii_letters as string of length 8 used as an + identity.""" + return "".join( + [random.choice(string.digits + string.ascii_letters) for x in range(8)] # nosec + ) + + +def response_for_text(username, text, id_string=None, identity=None): + """Processes an SMS ``text`` and returns the results.""" + if identity is None: + identity = random_identity() + + return process_incoming_smses( + username=username, id_string=id_string, incomings=[(identity, text)] + )[0] + + class TestBaseSMS(TestBase): + """ + TestBaseSMS - base class for sms_support test cases. + """ def setUp(self): TestBase.setUp(self) def setup_form(self, allow_sms=True): - self.id_string = 'sms_test_form' - self.sms_keyword = 'test' - self.username = 'auser' - self.password = 'auser' + """Helper method to setup an SMS form.""" + # pylint: disable=attribute-defined-outside-init + self.id_string = "sms_test_form" + self.sms_keyword = "test" + self.username = "auser" + self.password = "auser" self.this_directory = os.path.dirname(__file__) # init FH - self._create_user_and_login(username=self.username, - password=self.password) + self._create_user_and_login(username=self.username, password=self.password) # create a test form and activate SMS Support. - self._publish_xls_file_and_set_xform(os.path.join(self.this_directory, - 'fixtures', - 'sms_tutorial.xlsx')) + self._publish_xls_file_and_set_xform( + os.path.join(self.this_directory, "fixtures", "sms_tutorial.xlsx") + ) if allow_sms: xform = XForm.objects.get(id_string=self.id_string) xform.allows_sms = True xform.save() self.xform = xform - - def random_identity(self): - return ''.join([random.choice(string.digits + string.ascii_letters) - for x in range(8)]) - - def response_for_text(self, username, text, - id_string=None, identity=None): - if identity is None: - identity = self.random_identity() - - return process_incoming_smses(username=username, - id_string=None, - incomings=[(identity, text)])[0] diff --git a/onadata/apps/sms_support/tests/test_notallowed.py b/onadata/apps/sms_support/tests/test_notallowed.py index 85039336e1..0f2399f9cf 100644 --- a/onadata/apps/sms_support/tests/test_notallowed.py +++ b/onadata/apps/sms_support/tests/test_notallowed.py @@ -1,22 +1,20 @@ from __future__ import absolute_import -from onadata.apps.sms_support.tests.test_base_sms import TestBaseSMS +from onadata.apps.sms_support.tests.test_base_sms import TestBaseSMS, response_for_text from onadata.apps.sms_support.tools import SMS_SUBMISSION_REFUSED class TestNotAllowed(TestBaseSMS): - def setUp(self): TestBaseSMS.setUp(self) self.setup_form(allow_sms=False) def test_refused_not_enabled(self): # SMS submissions not allowed - result = self.response_for_text(self.username, 'test allo') - self.assertEqual(result['code'], SMS_SUBMISSION_REFUSED) + result = response_for_text(self.username, "test allo") + self.assertEqual(result["code"], SMS_SUBMISSION_REFUSED) def test_allow_sms(self): - result = self.response_for_text(self.username, - 'test +a 1 y 1950-02-22 john doe') - self.assertEqual(result['code'], SMS_SUBMISSION_REFUSED) - self.assertEqual(result.get('id'), None) + result = response_for_text(self.username, "test +a 1 y 1950-02-22 john doe") + self.assertEqual(result["code"], SMS_SUBMISSION_REFUSED) + self.assertEqual(result.get("id"), None) diff --git a/onadata/apps/sms_support/tests/test_parser.py b/onadata/apps/sms_support/tests/test_parser.py index b85bab4c98..7ec333959a 100644 --- a/onadata/apps/sms_support/tests/test_parser.py +++ b/onadata/apps/sms_support/tests/test_parser.py @@ -1,55 +1,52 @@ from __future__ import absolute_import -from onadata.apps.sms_support.tools import (SMS_API_ERROR, SMS_PARSING_ERROR, - SMS_SUBMISSION_ACCEPTED, - SMS_SUBMISSION_REFUSED) - -from onadata.apps.sms_support.tests.test_base_sms import TestBaseSMS +from onadata.apps.sms_support.tests.test_base_sms import TestBaseSMS, response_for_text +from onadata.apps.sms_support.tools import ( + SMS_API_ERROR, + SMS_PARSING_ERROR, + SMS_SUBMISSION_ACCEPTED, + SMS_SUBMISSION_REFUSED, +) class TestParser(TestBaseSMS): - def setUp(self): TestBaseSMS.setUp(self) self.setup_form(allow_sms=True) def test_api_error(self): # missing identity or text - result = self.response_for_text(self.username, 'hello', identity='') - self.assertEqual(result['code'], SMS_API_ERROR) + result = response_for_text(self.username, "hello", identity="") + self.assertEqual(result["code"], SMS_API_ERROR) - result = self.response_for_text(self.username, text='') - self.assertEqual(result['code'], SMS_API_ERROR) + result = response_for_text(self.username, text="") + self.assertEqual(result["code"], SMS_API_ERROR) def test_invalid_syntax(self): # invalid text message - result = self.response_for_text(self.username, 'hello') - self.assertEqual(result['code'], SMS_PARSING_ERROR) + result = response_for_text(self.username, "hello") + self.assertEqual(result["code"], SMS_PARSING_ERROR) def test_invalid_group(self): # invalid text message - result = self.response_for_text(self.username, '++a 20', - id_string=self.id_string) - self.assertEqual(result['code'], SMS_PARSING_ERROR) + result = response_for_text(self.username, "++a 20", id_string=self.id_string) + self.assertEqual(result["code"], SMS_PARSING_ERROR) def test_refused_with_keyword(self): # submission has proper keywrd with invalid text - result = self.response_for_text(self.username, 'test allo') - self.assertEqual(result['code'], SMS_PARSING_ERROR) + result = response_for_text(self.username, "test allo") + self.assertEqual(result["code"], SMS_PARSING_ERROR) def test_sucessful_submission(self): - result = self.response_for_text(self.username, - 'test +a 1 y 1950-02-22 john doe') - self.assertEqual(result['code'], SMS_SUBMISSION_ACCEPTED) - self.assertTrue(result['id']) + result = response_for_text(self.username, "test +a 1 y 1950-02-22 john doe") + self.assertEqual(result["code"], SMS_SUBMISSION_ACCEPTED) + self.assertTrue(result["id"]) def test_invalid_type(self): - result = self.response_for_text(self.username, - 'test +a yes y 1950-02-22 john doe') - self.assertEqual(result['code'], SMS_PARSING_ERROR) + result = response_for_text(self.username, "test +a yes y 1950-02-22 john doe") + self.assertEqual(result["code"], SMS_PARSING_ERROR) def test_missing_required_field(self): # required field name missing - result = self.response_for_text(self.username, - 'test +b ff') - self.assertEqual(result['code'], SMS_SUBMISSION_REFUSED) + result = response_for_text(self.username, "test +b ff") + self.assertEqual(result["code"], SMS_SUBMISSION_REFUSED) diff --git a/onadata/apps/sms_support/tools.py b/onadata/apps/sms_support/tools.py index f6af0471f8..033dc6cc89 100644 --- a/onadata/apps/sms_support/tools.py +++ b/onadata/apps/sms_support/tools.py @@ -1,63 +1,78 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu +""" +sms_support utility functions module. +""" import copy import io import mimetypes from xml.parsers.expat import ExpatError -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.files.uploadedfile import InMemoryUploadedFile from django.http import HttpRequest -from django.utils.translation import ugettext as _ -from past.builtins import basestring +from django.urls import reverse +from django.utils.translation import gettext as _ from onadata.apps.logger.models import XForm from onadata.apps.logger.models.instance import FormInactiveError -from onadata.apps.logger.xform_instance_parser import DuplicateInstance -from onadata.apps.logger.xform_instance_parser import InstanceEmptyError -from onadata.apps.logger.xform_instance_parser import InstanceInvalidUserError -from onadata.libs.utils.log import Actions -from onadata.libs.utils.log import audit_log +from onadata.apps.logger.xform_instance_parser import ( + DuplicateInstance, + InstanceEmptyError, + InstanceInvalidUserError, +) +from onadata.libs.utils.log import Actions, audit_log from onadata.libs.utils.logger_tools import create_instance -SMS_API_ERROR = 'SMS_API_ERROR' -SMS_PARSING_ERROR = 'SMS_PARSING_ERROR' -SMS_SUBMISSION_ACCEPTED = 'SMS_SUBMISSION_ACCEPTED' -SMS_SUBMISSION_REFUSED = 'SMS_SUBMISSION_REFUSED' -SMS_INTERNAL_ERROR = 'SMS_INTERNAL_ERROR' +SMS_API_ERROR = "SMS_API_ERROR" +SMS_PARSING_ERROR = "SMS_PARSING_ERROR" +SMS_SUBMISSION_ACCEPTED = "SMS_SUBMISSION_ACCEPTED" +SMS_SUBMISSION_REFUSED = "SMS_SUBMISSION_REFUSED" +SMS_INTERNAL_ERROR = "SMS_INTERNAL_ERROR" -BASE64_ALPHABET = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ' - 'abcdefghijklmnopqrstuvwxyz0123456789+/=') -DEFAULT_SEPARATOR = '+' +BASE64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" +DEFAULT_SEPARATOR = "+" DEFAULT_ALLOW_MEDIA = False -NA_VALUE = 'n/a' -BASE64_ALPHABET = None -META_FIELDS = ('start', 'end', 'today', 'deviceid', 'subscriberid', - 'imei', 'phonenumber') -MEDIA_TYPES = ('audio', 'video', 'photo') -DEFAULT_DATE_FORMAT = '%Y-%m-%d' -DEFAULT_DATETIME_FORMAT = '%Y-%m-%d-%H:%M' -SENSITIVE_FIELDS = ('text', 'select all that apply', 'geopoint', 'barcode') +NA_VALUE = "n/a" +META_FIELDS = ( + "start", + "end", + "today", + "deviceid", + "subscriberid", + "imei", + "phonenumber", +) +MEDIA_TYPES = ("audio", "video", "photo") +DEFAULT_DATE_FORMAT = "%Y-%m-%d" +DEFAULT_DATETIME_FORMAT = "%Y-%m-%d-%H:%M" +SENSITIVE_FIELDS = ("text", "select all that apply", "geopoint", "barcode") + +# pylint: disable=invalid-name +User = get_user_model() def is_last(index, items): - return index == len(items) - 1 or (items[-1].get('type') == 'note' and - index == len(items) - 2) + """Returns True if ``index`` is the last index in ``items``.""" + return index == len(items) - 1 or ( + items[-1].get("type") == "note" and index == len(items) - 2 + ) def get_sms_instance_id(instance): - """ Human-friendly unique ID of a submission for latter ref/update + """Human-friendly unique ID of a submission for latter ref/update - For now, we strip down to the first 8 chars of the UUID. - Until we figure out what we really want (might as well be used - by formhub XML) """ + For now, we strip down to the first 8 chars of the UUID. + Until we figure out what we really want (might as well be used + by formhub XML)""" return instance.uuid[:8] def sms_media_to_file(file_object, name): - if isinstance(file_object, basestring): + """Returns a file object from an SMS string.""" + if isinstance(file_object, str): file_object = io.BytesIO(file_object) def getsize(f): @@ -70,181 +85,197 @@ def getsize(f): name = name.strip() content_type, charset = mimetypes.guess_type(name) size = getsize(file_object) - return InMemoryUploadedFile(file=file_object, name=name, - field_name=None, content_type=content_type, - charset=charset, size=size) + return InMemoryUploadedFile( + file=file_object, + name=name, + field_name=None, + content_type=content_type, + charset=charset, + size=size, + ) -def generate_instance(username, xml_file, media_files, uuid=None): - ''' Process an XForm submission as if done via HTTP +# pylint: disable=too-many-return-statements +def generate_instance(username, xml_file, media_files, uuid=None): # noqa C901 + """Process an XForm submission as if done via HTTP - :param IO xml_file: file-like object containing XML XForm - :param string username: username of the Form's owner - :param list media_files: a list of UploadedFile objects - :param string uuid: an optionnal uuid for the instance. + :param IO xml_file: file-like object containing XML XForm + :param string username: username of the Form's owner + :param list media_files: a list of UploadedFile objects + :param string uuid: an optionnal uuid for the instance. - :returns a (status, message) tuple. ''' + :returns a (status, message) tuple.""" try: - instance = create_instance( - username, - xml_file, - media_files, - uuid=uuid - ) + instance = create_instance(username, xml_file, media_files, uuid=uuid) except InstanceInvalidUserError: - return {'code': SMS_SUBMISSION_REFUSED, - 'text': _(u"Username or ID required.")} + return {"code": SMS_SUBMISSION_REFUSED, "text": _("Username or ID required.")} except InstanceEmptyError: - return {'code': SMS_INTERNAL_ERROR, - 'text': _(u"Received empty submission. " - u"No instance was created")} + return { + "code": SMS_INTERNAL_ERROR, + "text": _("Received empty submission. No instance was created"), + } except FormInactiveError: - return {'code': SMS_SUBMISSION_REFUSED, - 'text': _(u"Form is not active")} + return {"code": SMS_SUBMISSION_REFUSED, "text": _("Form is not active")} except XForm.DoesNotExist: - return {'code': SMS_SUBMISSION_REFUSED, - 'text': _(u"Form does not exist on this account")} + return { + "code": SMS_SUBMISSION_REFUSED, + "text": _("Form does not exist on this account"), + } except ExpatError: - return {'code': SMS_INTERNAL_ERROR, - 'text': _(u"Improperly formatted XML.")} + return {"code": SMS_INTERNAL_ERROR, "text": _("Improperly formatted XML.")} except DuplicateInstance: - return {'code': SMS_SUBMISSION_REFUSED, - 'text': _(u"Duplicate submission")} + return {"code": SMS_SUBMISSION_REFUSED, "text": _("Duplicate submission")} if instance is None: - return {'code': SMS_INTERNAL_ERROR, - 'text': _(u"Unable to create submission.")} + return {"code": SMS_INTERNAL_ERROR, "text": _("Unable to create submission.")} user = User.objects.get(username=username) - audit = { - "xform": instance.xform.id_string - } - audit_log(Actions.SUBMISSION_CREATED, - user, instance.xform.user, - _("Created submission on form %(id_string)s.") % - {"id_string": instance.xform.id_string}, audit, HttpRequest()) + audit = {"xform": instance.xform.id_string} + audit_log( + Actions.SUBMISSION_CREATED, + user, + instance.xform.user, + _(f"Created submission on form {instance.xform.id_string}."), + audit, + HttpRequest(), + ) xml_file.close() - if len(media_files): - [_file.close() for _file in media_files] - - return {'code': SMS_SUBMISSION_ACCEPTED, - 'text': _(u"[SUCCESS] Your submission has been accepted."), - 'id': get_sms_instance_id(instance)} + if media_files: + for _file in media_files: + _file.close() + + return { + "code": SMS_SUBMISSION_ACCEPTED, + "text": _("[SUCCESS] Your submission has been accepted."), + "id": get_sms_instance_id(instance), + } -def is_sms_related(json_survey): - ''' Whether a form is considered to want sms Support +def is_sms_related(json_survey): # noqa C901 + """Whether a form is considered to want sms Support - return True if one sms-related field is defined. ''' + return True if one sms-related field is defined.""" - def treat(value, key=None): + def _treat(value, key=None): if key is None: return False - if key in ('sms_field', 'sms_option') and value: - if not value.lower() in ('no', 'false'): + if key in ("sms_field", "sms_option") and value: + if not value.lower() in ("no", "false"): return True + return False - def walk(dl): + def _walk(dl): if not isinstance(dl, (dict, list)): return False - iterator = [(None, e) for e in dl] \ - if isinstance(dl, list) else dl.items() + iterator = [(None, e) for e in dl] if isinstance(dl, list) else dl.items() for k, v in iterator: - if k == 'parent': + if k == "parent": continue - if treat(v, k): + if _treat(v, k): return True - if walk(v): + if _walk(v): return True return False - return walk(json_survey) + return _walk(json_survey) -def check_form_sms_compatibility(form, json_survey=None): - ''' Tests all SMS related rules on the XForm representation +# pylint: disable=too-many-locals,too-many-branches,too-many-statements +def check_form_sms_compatibility(form, json_survey=None): # noqa C901 + """Tests all SMS related rules on the XForm representation - Returns a view-compatible dict(type, text) with warnings or - a success message ''' + Returns a view-compatible dict(type, text) with warnings or + a success message""" if json_survey is None: - json_survey = form.get('form_o', {}) + json_survey = form.get("form_o", {}) def prep_return(msg, comp=None): - - from django.urls import reverse - - error = 'alert-info' - warning = 'alert-info' - success = 'alert-success' - outro = (u"
Please check the " - u"SMS Syntax Page." % {'syntax_url': reverse('syntax')}) + error = "alert-info" + warning = "alert-info" + success = "alert-success" + syntax_url = reverse("syntax") + outro = ( + f'
Please check the ' + "SMS Syntax Page." + ) # no compatibility at all if not comp: alert = error - msg = (u"%(prefix)s %(msg)s" - % {'prefix': u"Your Form is not SMS-compatible" - u". If you want to later enable " - u"SMS Support, please fix:
", - 'msg': msg}) + msg = "%(prefix)s %(msg)s" % { + "prefix": "Your Form is not SMS-compatible" + ". If you want to later enable " + "SMS Support, please fix:
", + "msg": msg, + } # no blocker but could be improved elif comp == 1: alert = warning - msg = (u"%(prefix)s
    %(msg)s
" - % {'prefix': u"Your form can be used with SMS, " - u"knowing that:", 'msg': msg}) + msg = "%(prefix)s
    %(msg)s
" % { + "prefix": "Your form can be used with SMS, knowing that:", + "msg": msg, + } # SMS compatible else: - outro = u"" + outro = "" alert = success - return {'type': alert, - 'text': u"%(msg)s%(outro)s" - % {'msg': msg, 'outro': outro}} + return { + "type": alert, + "text": "%(msg)s%(outro)s" % {"msg": msg, "outro": outro}, + } # first level children. should be groups - groups = json_survey.get('children', [{}]) + groups = json_survey.get("children", [{}]) # BLOCKERS # overload SENSITIVE_FIELDS if date or datetime format contain spaces. sensitive_fields = copy.copy(SENSITIVE_FIELDS) - date_format = json_survey.get('sms_date_format', DEFAULT_DATE_FORMAT) \ - or DEFAULT_DATE_FORMAT - datetime_format = json_survey.get('sms_datetime_format', - DEFAULT_DATETIME_FORMAT) \ + date_format = ( + json_survey.get("sms_date_format", DEFAULT_DATE_FORMAT) or DEFAULT_DATE_FORMAT + ) + datetime_format = ( + json_survey.get("sms_datetime_format", DEFAULT_DATETIME_FORMAT) or DEFAULT_DATETIME_FORMAT + ) if len(date_format.split()) > 1: - sensitive_fields += ('date', ) + sensitive_fields += ("date",) if len(datetime_format.split()) > 1: - sensitive_fields += ('datetime', ) + sensitive_fields += ("datetime",) # must not contain out-of-group questions - if sum([1 for e in groups if e.get('type') != 'group']): - return prep_return(_(u"All your questions must be in groups.")) + if sum([1 for e in groups if e.get("type") != "group"]): + return prep_return(_("All your questions must be in groups.")) # all groups must have an sms_field - bad_groups = [e.get('name') for e in groups - if not e.get('sms_field', '') and - not e.get('name', '') == 'meta'] + bad_groups = [ + e.get("name") + for e in groups + if not e.get("sms_field", "") and not e.get("name", "") == "meta" + ] if len(bad_groups): - return prep_return(_(u"All your groups must have an 'sms_field' " - u"(use 'meta' prefixed ones for non-fillable " - u"groups). %s" % bad_groups[-1])) + return prep_return( + _( + "All your groups must have an 'sms_field' " + "(use 'meta' prefixed ones for non-fillable " + f"groups). {bad_groups[-1]}" + ) + ) # all select_one or select_multiple fields muts have sms_option for each. for group in groups: - fields = group.get('children', [{}]) + fields = group.get("children", [{}]) for field in fields: - xlsf_type = field.get('type') - xlsf_name = field.get('name') - xlsf_choices = field.get('children') - if xlsf_type in ('select one', 'select all that apply'): + xlsf_type = field.get("type") + xlsf_name = field.get("name") + xlsf_choices = field.get("children") + if xlsf_type in ("select one", "select all that apply"): nb_choices = len(xlsf_choices) - options = list(set([c.get('sms_option', '') or None - for c in xlsf_choices])) + options = list( + set(c.get("sms_option", "") or None for c in xlsf_choices) + ) try: options.remove(None) except ValueError: @@ -252,75 +283,92 @@ def prep_return(msg, comp=None): nb_options = len(options) if nb_choices != nb_options: return prep_return( - _(u"Not all options in the choices list for " - u"%s have an " - u"sms_option value.") % xlsf_name + _( + "Not all options in the choices list for " + f"{xlsf_name} have an " + "sms_option value." + ) ) # has sensitive (space containing) fields in non-last position for group in groups: - fields = group.get('children', [{}]) + fields = group.get("children", [{}]) last_pos = len(fields) - 1 # consider last field to be last before note if there's a trailing note - if fields[last_pos].get('type') == 'note': + if fields[last_pos].get("type") == "note": if len(fields) - 1: last_pos -= 1 for idx, field in enumerate(fields): - if idx != last_pos and field.get('type', '') in sensitive_fields: - return prep_return(_(u"Questions for which values can contain " - u"spaces are only allowed on last " - u"position of group (%s)" - % field.get('name'))) + if idx != last_pos and field.get("type", "") in sensitive_fields: + return prep_return( + _( + "Questions for which values can contain " + "spaces are only allowed on last " + f"position of group ({field.get('name')})" + ) + ) # separator is not set or is within BASE64 alphabet and sms_allow_media - separator = json_survey.get('sms_separator', DEFAULT_SEPARATOR) \ - or DEFAULT_SEPARATOR - sms_allow_media = bool(json_survey.get('sms_allow_media', - DEFAULT_ALLOW_MEDIA) or DEFAULT_ALLOW_MEDIA) + separator = json_survey.get("sms_separator", DEFAULT_SEPARATOR) or DEFAULT_SEPARATOR + sms_allow_media = bool( + json_survey.get("sms_allow_media", DEFAULT_ALLOW_MEDIA) or DEFAULT_ALLOW_MEDIA + ) if sms_allow_media and separator in BASE64_ALPHABET: - return prep_return(_(u"When allowing medias ('sms_allow_media'), your " - u"separator (%s) must be outside Base64 alphabet " - u"(letters, digits and +/=). " - u"You case use '#' instead." % separator)) + return prep_return( + _( + "When allowing medias ('sms_allow_media'), your " + f"separator ({separator}) must be outside Base64 alphabet " + "(letters, digits and +/=). " + "You case use '#' instead." + ) + ) # WARNINGS warnings = [] # sms_separator not set - if not json_survey.get('sms_separator', ''): - warnings.append(u"
  • You have not set a separator. Default '+' " - u"separator will be used.
    • ") + if not json_survey.get("sms_separator", ""): + warnings.append( + "
  • You have not set a separator. Default '+' " + "separator will be used.
    • " + ) # has date field with no sms_date_format - if not json_survey.get('sms_date_format', ''): + if not json_survey.get("sms_date_format", ""): for group in groups: - if sum([1 for e in group.get('children', [{}]) - if e.get('type') == 'date']): - warnings.append(u"
  • You have 'date' fields without " - u"explicitly setting a date format. " - u"Default (%s) will be used.
    • " - % DEFAULT_DATE_FORMAT) + if sum([1 for e in group.get("children", [{}]) if e.get("type") == "date"]): + warnings.append( + "
  • You have 'date' fields without " + "explicitly setting a date format. " + f"Default ({DEFAULT_DATE_FORMAT}) will be used.
    • " + ) break # has datetime field with no datetime format - if not json_survey.get('sms_date_format', ''): + if not json_survey.get("sms_date_format", ""): for group in groups: - if sum([1 for e in group.get('children', [{}]) - if e.get('type') == 'datetime']): - warnings.append(u"
  • You have 'datetime' fields without " - u"explicitly setting a datetime format. " - u"Default (%s) will be used.
    • " - % DEFAULT_DATETIME_FORMAT) + if sum( + [1 for e in group.get("children", [{}]) if e.get("type") == "datetime"] + ): + warnings.append( + "
  • You have 'datetime' fields without " + "explicitly setting a datetime format. " + f"Default ({DEFAULT_DATETIME_FORMAT}) will be used.
    • " + ) break # date or datetime format contain space - if 'date' in sensitive_fields: - warnings.append(u"
  • 'sms_date_format' contains space which will " - u"require 'date' questions to be positioned at " - u"the end of groups (%s).
    • " % date_format) - if 'datetime' in sensitive_fields: - warnings.append(u"
  • 'sms_datetime_format' contains space which will " - u"require 'datetime' questions to be positioned at " - u"the end of groups (%s).
    • " % datetime_format) - - if len(warnings): - return prep_return(u"".join(warnings), comp=1) + if "date" in sensitive_fields: + warnings.append( + "
  • 'sms_date_format' contains space which will " + "require 'date' questions to be positioned at " + f"the end of groups ({date_format}).
    • " + ) + if "datetime" in sensitive_fields: + warnings.append( + "
  • 'sms_datetime_format' contains space which will " + "require 'datetime' questions to be positioned at " + f"the end of groups ({datetime_format}).
    • " + ) + + if warnings: + return prep_return("".join(warnings), comp=1) # Good to go - return prep_return(_(u"Note that your form is also SMS comptatible."), 2) + return prep_return(_("Note that your form is also SMS comptatible."), 2) diff --git a/onadata/apps/sms_support/views.py b/onadata/apps/sms_support/views.py index e28f2b410e..7d4f43e695 100644 --- a/onadata/apps/sms_support/views.py +++ b/onadata/apps/sms_support/views.py @@ -1,12 +1,15 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu +""" +sms_support views. +""" from __future__ import absolute_import import json -from django.http import HttpResponse -from django.utils.translation import ugettext as _ +from django.http import JsonResponse +from django.utils.translation import gettext as _ from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_GET, require_POST @@ -15,16 +18,20 @@ def get_response(data): - response = {'status': data.get('code'), - 'message': data.get('text'), - 'instanceID': data.get('id'), - 'sendouts': data.get('sendouts')} - return HttpResponse(json.dumps(response), content_type='application/json') + """Returns a JsonResponse object with `status`, `message`, `instanceID` and + `sendouts` based on the input ``data`` object.""" + response = { + "status": data.get("code"), + "message": data.get("text"), + "instanceID": data.get("id"), + "sendouts": data.get("sendouts"), + } + return JsonResponse(response) @require_GET def import_submission(request, username): - """ Process an SMS text as a form submission + """Process an SMS text as a form submission :param string identity: phone number of the sender :param string text: SMS content @@ -34,7 +41,7 @@ def import_submission(request, username): 'message': Error message if not ACCEPTED. 'id: Unique submission ID if ACCEPTED. - """ + """ return import_submission_for_form(request, username, None) @@ -42,46 +49,57 @@ def import_submission(request, username): @require_POST @csrf_exempt def import_multiple_submissions(request, username): - ''' Process several POSTED SMS texts as XForm submissions + """Process several POSTED SMS texts as XForm submissions :param json messages: JSON list of {"identity": "x", "text": "x"} :returns json list of {"status": "x", "message": "x", "id": "x"} - ''' + """ return import_multiple_submissions_for_form(request, username, None) @require_GET def import_submission_for_form(request, username, id_string): - """ idem import_submission with a defined id_string """ + """idem import_submission with a defined id_string""" - sms_identity = request.GET.get('identity', '').strip() - sms_text = request.GET.get('text', '').strip() + sms_identity = request.GET.get("identity", "").strip() + sms_text = request.GET.get("text", "").strip() if not sms_identity or not sms_text: - return get_response({'code': SMS_API_ERROR, - 'text': _(u"`identity` and `message` are " - u"both required and must not be " - u"empty.")}) + return get_response( + { + "code": SMS_API_ERROR, + "text": _( + "`identity` and `message` are " + "both required and must not be " + "empty." + ), + } + ) incomings = [(sms_identity, sms_text)] response = process_incoming_smses(username, incomings, id_string)[-1] return get_response(response) +# pylint: disable=invalid-name @require_POST @csrf_exempt def import_multiple_submissions_for_form(request, username, id_string): - """ idem import_multiple_submissions with a defined id_string """ - - messages = json.loads(request.POST.get('messages', '[]')) - incomings = [(m.get('identity', ''), m.get('text', '')) for m in messages] - - responses = [{'status': d.get('code'), - 'message': d.get('text'), - 'instanceID': d.get('id'), - 'sendouts': d.get('sendouts')} for d - in process_incoming_smses(username, incomings, id_string)] - - return HttpResponse(json.dumps(responses), content_type='application/json') + """idem import_multiple_submissions with a defined id_string""" + + messages = json.loads(request.POST.get("messages", "[]")) + incomings = [(m.get("identity", ""), m.get("text", "")) for m in messages] + + responses = [ + { + "status": d.get("code"), + "message": d.get("text"), + "instanceID": d.get("id"), + "sendouts": d.get("sendouts"), + } + for d in process_incoming_smses(username, incomings, id_string) + ] + + return JsonResponse(responses, safe=False) diff --git a/onadata/apps/viewer/management/commands/import.py b/onadata/apps/viewer/management/commands/import.py index 7c6dc3a25d..64a04925f9 100644 --- a/onadata/apps/viewer/management/commands/import.py +++ b/onadata/apps/viewer/management/commands/import.py @@ -1,16 +1,21 @@ #!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=4 coding=utf-8 +# vim: ai ts=4 sts=4 et sw=4 +# -*- coding: utf-8 -*- +"""import command - Combines and runs import_forms and import_instances commands""" import os from django.core.management.base import BaseCommand from django.core.management import call_command -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy class Command(BaseCommand): - help = ugettext_lazy("Import ODK forms and instances.") + """Import ODK forms and instances.""" + + help = gettext_lazy("Import ODK forms and instances.") def handle(self, *args, **kwargs): + """Import ODK forms and instances.""" path = args[0] - call_command('import_forms', os.path.join(path, "forms")) - call_command('import_instances', os.path.join(path, "instances")) + call_command("import_forms", os.path.join(path, "forms")) + call_command("import_instances", os.path.join(path, "instances")) diff --git a/onadata/apps/viewer/management/commands/import_forms.py b/onadata/apps/viewer/management/commands/import_forms.py deleted file mode 100644 index aed6bc40ce..0000000000 --- a/onadata/apps/viewer/management/commands/import_forms.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=4 coding=utf-8 -from __future__ import absolute_import - -import glob -import os - -from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy - -from onadata.apps.logger.models import XForm - - -class Command(BaseCommand): - help = ugettext_lazy("Import a folder of XForms for ODK.") - - def handle(self, *args, **kwargs): - path = args[0] - for form in glob.glob(os.path.join(path, "*")): - f = open(form) - XForm.objects.get_or_create(xml=f.read(), active=False) - f.close() diff --git a/onadata/apps/viewer/management/commands/mark_start_times.py b/onadata/apps/viewer/management/commands/mark_start_times.py index e46624f02b..096fab9b39 100644 --- a/onadata/apps/viewer/management/commands/mark_start_times.py +++ b/onadata/apps/viewer/management/commands/mark_start_times.py @@ -1,19 +1,20 @@ from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy, ugettext as _ +from django.utils.translation import gettext_lazy, gettext as _ from onadata.apps.viewer.models.data_dictionary import DataDictionary class Command(BaseCommand): - help = ugettext_lazy("This is a one-time command to " - "mark start times of old surveys.") + help = gettext_lazy( + "This is a one-time command to " "mark start times of old surveys." + ) def handle(self, *args, **kwargs): for dd in DataDictionary.objects.all(): try: - dd._mark_start_time_boolean() + dd.mark_start_time_boolean() dd.save() except Exception: - self.stderr.write(_( - "Could not mark start time for DD: %(data)s") % { - 'data': repr(dd)}) + self.stderr.write( + _("Could not mark start time for DD: %(data)s") % {"data": repr(dd)} + ) diff --git a/onadata/apps/viewer/management/commands/set_uuid_in_xml.py b/onadata/apps/viewer/management/commands/set_uuid_in_xml.py index 532bd05224..0faf46ed7b 100644 --- a/onadata/apps/viewer/management/commands/set_uuid_in_xml.py +++ b/onadata/apps/viewer/management/commands/set_uuid_in_xml.py @@ -1,12 +1,12 @@ from django.core.management.base import BaseCommand -from django.utils.translation import ugettext as _, ugettext_lazy +from django.utils.translation import gettext as _, gettext_lazy from onadata.apps.viewer.models.data_dictionary import DataDictionary from onadata.libs.utils.model_tools import queryset_iterator class Command(BaseCommand): - help = ugettext_lazy("Insert UUID into XML of all existing XForms") + help = gettext_lazy("Insert UUID into XML of all existing XForms") def handle(self, *args, **kwargs): self.stdout.write(_('%(nb)d XForms to update') @@ -14,7 +14,7 @@ def handle(self, *args, **kwargs): for i, dd in enumerate( queryset_iterator(DataDictionary.objects.all())): if dd.xls: - dd._set_uuid_in_xml() + dd.set_uuid_in_xml() super(DataDictionary, dd).save() if (i + 1) % 10 == 0: self.stdout.write(_('Updated %(nb)d XForms...') % {'nb': i}) diff --git a/onadata/apps/viewer/migrations/0001_pre-django-3-upgrade.py b/onadata/apps/viewer/migrations/0001_pre-django-3-upgrade.py new file mode 100644 index 0000000000..b4ce22c289 --- /dev/null +++ b/onadata/apps/viewer/migrations/0001_pre-django-3-upgrade.py @@ -0,0 +1,66 @@ +# Generated by Django 3.2.13 on 2022-04-25 08:11 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + replaces = [('viewer', '0001_initial'), ('viewer', '0002_export_options'), ('viewer', '0003_auto_20151226_0100'), ('viewer', '0004_auto_20151226_0109'), ('viewer', '0005_auto_20160408_0325'), ('viewer', '0006_auto_20160418_0525'), ('viewer', '0007_export_error_message'), ('viewer', '0008_auto_20190125_0517'), ('viewer', '0009_alter_export_options')] + + initial = True + + dependencies = [ + ('logger', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='ColumnRename', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('xpath', models.CharField(max_length=255, unique=True)), + ('column_name', models.CharField(max_length=32)), + ], + ), + migrations.CreateModel( + name='ParsedInstance', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('start_time', models.DateTimeField(null=True)), + ('end_time', models.DateTimeField(null=True)), + ('lat', models.FloatField(null=True)), + ('lng', models.FloatField(null=True)), + ('instance', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='parsed_instance', to='logger.instance')), + ], + ), + migrations.CreateModel( + name='DataDictionary', + fields=[ + ], + options={ + 'proxy': True, + }, + bases=('logger.xform',), + ), + migrations.CreateModel( + name='Export', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_on', models.DateTimeField(auto_now_add=True)), + ('filename', models.CharField(blank=True, max_length=255, null=True)), + ('filedir', models.CharField(blank=True, max_length=255, null=True)), + ('export_type', models.CharField(choices=[('xls', 'Excel'), ('csv', 'CSV'), ('zip', 'ZIP'), ('kml', 'kml'), ('csv_zip', 'CSV ZIP'), ('sav_zip', 'SAV ZIP'), ('sav', 'SAV'), ('external', 'Excel'), ('osm', 'osm'), ('gsheets', 'Google Sheets')], default='xls', max_length=10)), + ('task_id', models.CharField(blank=True, max_length=255, null=True)), + ('time_of_last_submission', models.DateTimeField(default=None, null=True)), + ('internal_status', models.SmallIntegerField(default=0)), + ('export_url', models.URLField(default=None, null=True)), + ('xform', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='logger.xform')), + ('options', models.JSONField(default=dict)), + ('error_message', models.CharField(blank=True, max_length=255, null=True)), + ], + options={ + 'unique_together': {('xform', 'filename')}, + }, + ), + ] diff --git a/onadata/apps/viewer/migrations/0002_export_options.py b/onadata/apps/viewer/migrations/0002_export_options.py index 8da8deafe7..f7eb4f77d5 100644 --- a/onadata/apps/viewer/migrations/0002_export_options.py +++ b/onadata/apps/viewer/migrations/0002_export_options.py @@ -1,21 +1,20 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -from django.db import migrations -import jsonfield.fields +from django.db import models, migrations class Migration(migrations.Migration): dependencies = [ - ('viewer', '0001_initial'), + ("viewer", "0001_initial"), ] operations = [ migrations.AddField( - model_name='export', - name='options', - field=jsonfield.fields.JSONField(default={}), + model_name="export", + name="options", + field=models.JSONField(default={}), preserve_default=True, ), ] diff --git a/onadata/apps/viewer/migrations/0009_alter_export_options.py b/onadata/apps/viewer/migrations/0009_alter_export_options.py new file mode 100644 index 0000000000..d703c213fd --- /dev/null +++ b/onadata/apps/viewer/migrations/0009_alter_export_options.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-04-25 07:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0008_auto_20190125_0517'), + ] + + operations = [ + migrations.AlterField( + model_name='export', + name='options', + field=models.JSONField(default=dict), + ), + ] diff --git a/onadata/apps/viewer/models/__init__.py b/onadata/apps/viewer/models/__init__.py index 9d4fc352e4..fb1341abfd 100644 --- a/onadata/apps/viewer/models/__init__.py +++ b/onadata/apps/viewer/models/__init__.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Viewer models. +""" from onadata.apps.viewer.models.parsed_instance import ParsedInstance # noqa from onadata.apps.viewer.models.data_dictionary import DataDictionary # noqa from onadata.apps.viewer.models.export import Export # noqa diff --git a/onadata/apps/viewer/models/column_rename.py b/onadata/apps/viewer/models/column_rename.py index 5e5efd17e0..7074460751 100644 --- a/onadata/apps/viewer/models/column_rename.py +++ b/onadata/apps/viewer/models/column_rename.py @@ -1,7 +1,15 @@ +# -*- coding: utf-8 -*- +""" +ColumnRename model +""" from django.db import models class ColumnRename(models.Model): + """ + ColumnRename model + """ + xpath = models.CharField(max_length=255, unique=True) column_name = models.CharField(max_length=32) @@ -10,4 +18,5 @@ class Meta: @classmethod def get_dict(cls): - return dict([(cr.xpath, cr.column_name) for cr in cls.objects.all()]) + """Returns a dictionary where xpath is key and column_name is value""" + return {cr.xpath: cr.column_name for cr in cls.objects.all()} diff --git a/onadata/apps/viewer/models/data_dictionary.py b/onadata/apps/viewer/models/data_dictionary.py index 2e314eb02e..2b77106fb4 100644 --- a/onadata/apps/viewer/models/data_dictionary.py +++ b/onadata/apps/viewer/models/data_dictionary.py @@ -6,15 +6,11 @@ from io import BytesIO, StringIO import unicodecsv as csv -import xlrd import openpyxl -from builtins import str as text -from django.core.files.storage import get_storage_class from django.core.files.uploadedfile import InMemoryUploadedFile from django.db.models.signals import post_save, pre_save from django.utils import timezone -from django.utils.encoding import python_2_unicode_compatible -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from floip import FloipSurvey from kombu.exceptions import OperationalError from pyxform.builder import create_survey_element_from_dict @@ -40,7 +36,7 @@ def is_newline_error(e): "new-line character seen in unquoted field - do you need" " to open the file in universal-newline mode?" ) - return newline_error == text(e) + return newline_error == str(e) def process_xlsform(xls, default_name): @@ -54,10 +50,10 @@ def process_xlsform(xls, default_name): file_object = xls if xls.name.endswith("csv"): file_object = None - if not get_storage_class()().exists(xls.path): + if not isinstance(xls.name, InMemoryUploadedFile): file_object = StringIO(xls.read().decode("utf-8")) try: - return parse_file_to_json(xls.path, file_object=file_object) + return parse_file_to_json(xls.name, file_object=file_object) except csv.Error as e: if is_newline_error(e): xls.seek(0) @@ -120,7 +116,7 @@ def sheet_to_csv(xls_content, sheet_name): except ValueError: pass elif sheet.cell(row, index).is_date: - val = xlrd.xldate_as_datetime(val, workbook.datemode).isoformat() + val = val.strftime("%Y-%m-%d").isoformat() row_values.append(val) writer.writerow([v for v, m in zip(row_values, mask) if m]) else: @@ -138,7 +134,6 @@ def upload_to(instance, filename, username=None): return os.path.join(username, "xls", os.path.split(filename)[1]) -@python_2_unicode_compatible class DataDictionary(XForm): # pylint: disable=too-many-instance-attributes """ DataDictionary model class. @@ -192,15 +187,15 @@ def save(self, *args, **kwargs): survey["name"] = default_name else: survey["id_string"] = self.id_string - self.json = survey.to_json() + self.json = survey.to_json_dict() self.xml = survey.to_xml() self.version = survey.get("version") self.last_updated_at = timezone.now() self.title = survey.get("title") - self._mark_start_time_boolean() + self.mark_start_time_boolean() set_uuid(self) - self._set_uuid_in_xml() - self._set_hash() + self.set_uuid_in_xml() + self.set_hash() if "skip_xls_read" in kwargs: del kwargs["skip_xls_read"] diff --git a/onadata/apps/viewer/models/export.py b/onadata/apps/viewer/models/export.py index 01f8dfde41..4e28916c31 100644 --- a/onadata/apps/viewer/models/export.py +++ b/onadata/apps/viewer/models/export.py @@ -3,19 +3,19 @@ Export model. """ import os -from future.utils import python_2_unicode_compatible + from tempfile import NamedTemporaryFile from django.core.files.storage import get_storage_class -from django.contrib.postgres.fields import JSONField from django.db import models +from django.db.models import JSONField from django.db.models.signals import post_delete -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from onadata.libs.utils.common_tags import OSM from onadata.libs.utils import async_status -EXPORT_QUERY_KEY = 'query' +EXPORT_QUERY_KEY = "query" # pylint: disable=unused-argument @@ -23,7 +23,7 @@ def export_delete_callback(sender, **kwargs): """ Delete export file when an export object is deleted. """ - export = kwargs['instance'] + export = kwargs["instance"] storage = get_storage_class()() if export.filepath and storage.exists(export.filepath): storage.delete(export.filepath) @@ -38,69 +38,69 @@ def get_export_options_query_kwargs(options): if field in options: field_value = options.get(field) - key = 'options__{}'.format(field) + key = "options__{}".format(field) options_kwargs[key] = field_value return options_kwargs -@python_2_unicode_compatible class ExportTypeError(Exception): """ ExportTypeError exception class. """ + def __str__(self): - return _(u'Invalid export type specified') + return _("Invalid export type specified") -@python_2_unicode_compatible class ExportConnectionError(Exception): """ ExportConnectionError exception class. """ + def __str__(self): - return _(u'Export server is down.') + return _("Export server is down.") -@python_2_unicode_compatible class Export(models.Model): """ Class representing a data export from an XForm """ - XLS_EXPORT = 'xls' - CSV_EXPORT = 'csv' - KML_EXPORT = 'kml' - ZIP_EXPORT = 'zip' - CSV_ZIP_EXPORT = 'csv_zip' - SAV_ZIP_EXPORT = 'sav_zip' - SAV_EXPORT = 'sav' - EXTERNAL_EXPORT = 'external' + + XLS_EXPORT = "xls" + CSV_EXPORT = "csv" + KML_EXPORT = "kml" + ZIP_EXPORT = "zip" + CSV_ZIP_EXPORT = "csv_zip" + SAV_ZIP_EXPORT = "sav_zip" + SAV_EXPORT = "sav" + EXTERNAL_EXPORT = "external" OSM_EXPORT = OSM - GOOGLE_SHEETS_EXPORT = 'gsheets' + GOOGLE_SHEETS_EXPORT = "gsheets" EXPORT_MIMES = { - 'xls': 'vnd.ms-excel', - 'xlsx': 'vnd.openxmlformats', - 'csv': 'csv', - 'zip': 'zip', - 'csv_zip': 'zip', - 'sav_zip': 'zip', - 'sav': 'sav', - 'kml': 'vnd.google-earth.kml+xml', - OSM: OSM + "xls": "vnd.ms-excel", + "xlsx": "vnd.openxmlformats", + "csv": "csv", + "zip": "zip", + "csv_zip": "zip", + "sav_zip": "zip", + "sav": "sav", + "kml": "vnd.google-earth.kml+xml", + OSM: OSM, } EXPORT_TYPES = [ - (XLS_EXPORT, 'Excel'), - (CSV_EXPORT, 'CSV'), - (ZIP_EXPORT, 'ZIP'), - (KML_EXPORT, 'kml'), - (CSV_ZIP_EXPORT, 'CSV ZIP'), - (SAV_ZIP_EXPORT, 'SAV ZIP'), - (SAV_EXPORT, 'SAV'), - (EXTERNAL_EXPORT, 'Excel'), + (XLS_EXPORT, "Excel"), + (CSV_EXPORT, "CSV"), + (ZIP_EXPORT, "ZIP"), + (KML_EXPORT, "kml"), + (CSV_ZIP_EXPORT, "CSV ZIP"), + (SAV_ZIP_EXPORT, "SAV ZIP"), + (SAV_EXPORT, "SAV"), + (EXTERNAL_EXPORT, "Excel"), (OSM, OSM), - (GOOGLE_SHEETS_EXPORT, 'Google Sheets'), + (GOOGLE_SHEETS_EXPORT, "Google Sheets"), ] EXPORT_OPTION_FIELDS = [ @@ -131,7 +131,7 @@ class Export(models.Model): MAX_EXPORTS = 10 # Required fields - xform = models.ForeignKey('logger.XForm', on_delete=models.CASCADE) + xform = models.ForeignKey("logger.XForm", on_delete=models.CASCADE) export_type = models.CharField( max_length=10, choices=EXPORT_TYPES, default=XLS_EXPORT ) @@ -159,14 +159,15 @@ class Meta: unique_together = (("xform", "filename"),) def __str__(self): - return u'%s - %s (%s)' % (self.export_type, self.xform, self.filename) + return "%s - %s (%s)" % (self.export_type, self.xform, self.filename) def save(self, *args, **kwargs): # pylint: disable=arguments-differ if not self.pk and self.xform: # if new, check if we've hit our limit for exports for this form, # if so, delete oldest num_existing_exports = Export.objects.filter( - xform=self.xform, export_type=self.export_type).count() + xform=self.xform, export_type=self.export_type + ).count() if num_existing_exports >= self.MAX_EXPORTS: Export._delete_oldest_export(self.xform, self.export_type) @@ -174,8 +175,7 @@ def save(self, *args, **kwargs): # pylint: disable=arguments-differ # update time_of_last_submission with # xform.time_of_last_submission_update # pylint: disable=E1101 - self.time_of_last_submission = self.xform.\ - time_of_last_submission_update() + self.time_of_last_submission = self.xform.time_of_last_submission_update() if self.filename: self.internal_status = Export.SUCCESSFUL super(Export, self).save(*args, **kwargs) @@ -183,7 +183,8 @@ def save(self, *args, **kwargs): # pylint: disable=arguments-differ @classmethod def _delete_oldest_export(cls, xform, export_type): oldest_export = Export.objects.filter( - xform=xform, export_type=export_type).order_by('created_on')[0] + xform=xform, export_type=export_type + ).order_by("created_on")[0] oldest_export.delete() @property @@ -227,9 +228,9 @@ def _update_filedir(self): if not self.filename: raise AssertionError() # pylint: disable=E1101 - self.filedir = os.path.join(self.xform.user.username, - 'exports', self.xform.id_string, - self.export_type) + self.filedir = os.path.join( + self.xform.user.username, "exports", self.xform.id_string, self.export_type + ) @property def filepath(self): @@ -273,16 +274,22 @@ def exports_outdated(cls, xform, export_type, options=None): try: export_options = get_export_options_query_kwargs(options) latest_export = Export.objects.filter( - xform=xform, export_type=export_type, + xform=xform, + export_type=export_type, internal_status__in=[Export.SUCCESSFUL, Export.PENDING], - **export_options).latest('created_on') + **export_options + ).latest("created_on") except cls.DoesNotExist: return True else: - if latest_export.time_of_last_submission is not None \ - and xform.time_of_last_submission_update() is not None: - return latest_export.time_of_last_submission <\ - xform.time_of_last_submission_update() + if ( + latest_export.time_of_last_submission is not None + and xform.time_of_last_submission_update() is not None + ): + return ( + latest_export.time_of_last_submission + < xform.time_of_last_submission_update() + ) # return true if we can't determine the status, to force # auto-generation @@ -293,8 +300,7 @@ def is_filename_unique(cls, xform, filename): """ Return True if the filename is unique. """ - return Export.objects.filter( - xform=xform, filename=filename).count() == 0 + return Export.objects.filter(xform=xform, filename=filename).count() == 0 post_delete.connect(export_delete_callback, sender=Export) diff --git a/onadata/apps/viewer/models/parsed_instance.py b/onadata/apps/viewer/models/parsed_instance.py index 30bd1e2b4d..7984695898 100644 --- a/onadata/apps/viewer/models/parsed_instance.py +++ b/onadata/apps/viewer/models/parsed_instance.py @@ -1,166 +1,206 @@ +# -*- coding: utf-8 -*- +""" +ParsedInstance model +""" import datetime import json import types -from builtins import str as text -import six -from dateutil import parser from django.conf import settings -from django.db import connection -from django.db import models +from django.db import connection, models from django.db.models.query import EmptyQuerySet -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ + +import six +from dateutil import parser -from onadata.apps.logger.models.instance import Instance -from onadata.apps.logger.models.instance import _get_attachments_from_instance +from onadata.apps.logger.models.instance import Instance, _get_attachments_from_instance from onadata.apps.logger.models.note import Note from onadata.apps.logger.models.xform import _encode_for_mongo -from onadata.apps.viewer.parsed_instance_tools import (get_where_clause, - NONE_JSON_FIELDS) +from onadata.apps.viewer.parsed_instance_tools import NONE_JSON_FIELDS, get_where_clause from onadata.libs.models.sorting import ( - json_order_by, json_order_by_params, sort_from_mongo_sort_str) -from onadata.libs.utils.common_tags import ID, UUID, ATTACHMENTS, \ - GEOLOCATION, SUBMISSION_TIME, MONGO_STRFTIME, BAMBOO_DATASET_ID, \ - DELETEDAT, TAGS, NOTES, SUBMITTED_BY, VERSION, DURATION, EDITED, \ - MEDIA_COUNT, TOTAL_MEDIA, MEDIA_ALL_RECEIVED, XFORM_ID, REVIEW_STATUS, \ - REVIEW_COMMENT, DATE_MODIFIED, REVIEW_DATE + json_order_by, + json_order_by_params, + sort_from_mongo_sort_str, +) +from onadata.libs.utils.common_tags import ( + ATTACHMENTS, + BAMBOO_DATASET_ID, + DATE_MODIFIED, + DELETEDAT, + DURATION, + EDITED, + GEOLOCATION, + ID, + MEDIA_ALL_RECEIVED, + MEDIA_COUNT, + MONGO_STRFTIME, + NOTES, + REVIEW_COMMENT, + REVIEW_DATE, + REVIEW_STATUS, + SUBMISSION_TIME, + SUBMITTED_BY, + TAGS, + TOTAL_MEDIA, + UUID, + VERSION, + XFORM_ID, +) from onadata.libs.utils.model_tools import queryset_iterator from onadata.libs.utils.mongo import _is_invalid_for_mongo -DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S' +DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S" class ParseError(Exception): - pass + """ + Raise when an exception happens when parsing the XForm XML submission. + """ def datetime_from_str(text): + """ + Parses a datetime from a string and returns the datetime object. + """ # Assumes text looks like 2011-01-01T09:50:06.966 if text is None: return None - dt = None try: - dt = parser.parse(text) - except Exception: + return parser.parse(text) + except (TypeError, ValueError): return None - return dt + return None -def dict_for_mongo(d): - for key, value in d.items(): +def dict_for_mongo(item): + """ + Validates the keys of a python object. + """ + for key, value in item.items(): if isinstance(value, list): - value = [dict_for_mongo(e) - if isinstance(e, dict) else e for e in value] + value = [dict_for_mongo(e) if isinstance(e, dict) else e for e in value] elif isinstance(value, dict): value = dict_for_mongo(value) - elif key == '_id': + elif key == "_id": try: - d[key] = int(value) + item[key] = int(value) except ValueError: # if it is not an int don't convert it pass if _is_invalid_for_mongo(key): - del d[key] - d[_encode_for_mongo(key)] = value - return d + del item[key] + item[_encode_for_mongo(key)] = value + return item def get_name_from_survey_element(element): + """ + Returns the abbreviated xpath of an element. + """ return element.get_abbreviated_xpath() def _parse_sort_fields(fields): for field in fields: - key = field.lstrip('-') - if field.startswith('-') and key in NONE_JSON_FIELDS.keys(): + key = field.lstrip("-") + if field.startswith("-") and key in NONE_JSON_FIELDS: field = NONE_JSON_FIELDS.get(key) - yield f'-{field}' + yield f"-{field}" else: yield NONE_JSON_FIELDS.get(field, field) -def _query_iterator(sql, fields=None, params=[], count=False): +def _query_iterator(sql, fields=None, params=None, count=False): + def parse_json(data): + try: + return json.loads(data) + except ValueError: + return data + if not sql: - raise ValueError(_(u"Bad SQL: %s" % sql)) + raise ValueError(_(f"Bad SQL: {sql}")) + params = [] if params is None else params cursor = connection.cursor() sql_params = fields + params if fields is not None else params if count: # do sql count of subquery, takes into account all options sql has and # is less hacky - sql = u"SELECT COUNT(*) FROM (" + sql + ") AS CQ" - fields = [u'count'] - - cursor.execute(sql, [text(i) for i in sql_params]) + sql = "SELECT COUNT(*) FROM (" + sql + ") AS CQ" + fields = ["count"] + cursor.execute(sql, [str(i) for i in sql_params]) if fields is None: for row in cursor.fetchall(): - yield row[0] + yield parse_json(row[0]) if row[0] else None else: for row in cursor.fetchall(): yield dict(zip(fields, row)) def get_etag_hash_from_query(queryset, sql=None, params=None): - """Returns md5 hash from the date_modified field or - """ + """Returns md5 hash from the date_modified field or""" if not isinstance(queryset, EmptyQuerySet): if sql is None: sql, params = queryset.query.sql_with_params() sql = ( "SELECT md5(string_agg(date_modified::text, ''))" - " FROM (SELECT date_modified " + sql[sql.find('FROM '):] + ") AS A" + " FROM (SELECT date_modified " + sql[sql.find("FROM ") :] + ") AS A" ) - etag_hash = [i for i in _query_iterator(sql, params=params) - if i is not None] + etag_hash = [i for i in _query_iterator(sql, params=params) if i is not None] if etag_hash: return etag_hash[0] - return u'%s' % datetime.datetime.utcnow() + return f"{datetime.datetime.utcnow()}" +# pylint: disable=too-many-arguments def _start_index_limit(records, sql, fields, params, sort, start_index, limit): - if start_index is not None and \ - (start_index < 0 or (limit is not None and limit < 0)): + if (start_index is not None and start_index < 0) or ( + limit is not None and limit < 0 + ): raise ValueError(_("Invalid start/limit params")) if (start_index is not None or limit is not None) and not sql: sql, params = records.query.sql_with_params() params = list(params) - start_index = 0 \ - if limit is not None and start_index is None else start_index - - if start_index is not None and \ - (ParsedInstance._has_json_fields(sort) or fields): + start_index = 0 if limit is not None and start_index is None else start_index + # pylint: disable=protected-access + has_json_fields = ParsedInstance._has_json_fields(sort) + if start_index is not None and (has_json_fields or fields): params += [start_index] - sql = u"%s OFFSET %%s" % sql - if limit is not None and \ - (ParsedInstance._has_json_fields(sort) or fields): - sql = u"%s LIMIT %%s" % sql + sql = f"{sql} OFFSET %s" + if limit is not None and (has_json_fields or fields): + sql = f"{sql} LIMIT %s" params += [limit] - if start_index is not None and limit is not None and not fields and \ - not ParsedInstance._has_json_fields(sort): - records = records[start_index: start_index + limit] - if start_index is not None and limit is None and not fields and \ - not ParsedInstance._has_json_fields(sort): + if ( + start_index is not None + and limit is not None + and not fields + and not has_json_fields + ): + records = records[start_index : start_index + limit] + if start_index is not None and limit is None and not fields and not has_json_fields: records = records[start_index:] return records, sql, params def _get_instances(xform, start, end): - kwargs = {'deleted_at': None} + kwargs = {"deleted_at": None} if isinstance(start, datetime.datetime): - kwargs.update({'date_created__gte': start}) + kwargs.update({"date_created__gte": start}) if isinstance(end, datetime.datetime): - kwargs.update({'date_created__lte': end}) + kwargs.update({"date_created__lte": end}) if xform.is_merged_dataset: - xforms = xform.mergedxform.xforms.filter(deleted_at__isnull=True)\ - .values_list('id', flat=True) - xform_ids = [i for i in xforms] or [xform.pk] + xforms = xform.mergedxform.xforms.filter(deleted_at__isnull=True).values_list( + "id", flat=True + ) + xform_ids = list(xforms) or [xform.pk] instances = Instance.objects.filter(xform_id__in=xform_ids) else: instances = xform.instances @@ -169,14 +209,27 @@ def _get_instances(xform, start, end): def _get_sort_fields(sort): - sort = ['id'] if sort is None else sort_from_mongo_sort_str(sort) - - return [i for i in _parse_sort_fields(sort)] - - -def get_sql_with_params(xform, query=None, fields=None, sort=None, start=None, - end=None, start_index=None, limit=None, count=None, - json_only: bool = True): + sort = ["id"] if sort is None else sort_from_mongo_sort_str(sort) + + return list(_parse_sort_fields(sort)) + + +# pylint: disable=too-many-locals +def get_sql_with_params( + xform, + query=None, + fields=None, + sort=None, + start=None, + end=None, + start_index=None, + limit=None, + count=None, + json_only: bool = True, +): + """ + Returns the SQL and related parameters. + """ records = _get_instances(xform, start, end) params = [] sort = _get_sort_fields(sort) @@ -184,31 +237,31 @@ def get_sql_with_params(xform, query=None, fields=None, sort=None, start=None, known_integers = [ get_name_from_survey_element(e) - for e in xform.get_survey_elements_of_type('integer')] + for e in xform.get_survey_elements_of_type("integer") + ] where, where_params = get_where_clause(query, known_integers) if fields and isinstance(fields, six.string_types): fields = json.loads(fields) if fields: - field_list = [u"json->%s" for _i in fields] - sql = u"SELECT %s FROM logger_instance" % u",".join(field_list) + field_list = ["json->%s" for _i in fields] + sql = f"SELECT {','.join(field_list)} FROM logger_instance" - sql_where = u"" + sql_where = "" if where_params: - sql_where = u" AND " + u" AND ".join(where) + sql_where = " AND " + " AND ".join(where) - sql += u" WHERE xform_id = %s " + sql_where \ - + u" AND deleted_at IS NULL" + sql += " WHERE xform_id = %s " + sql_where + " AND deleted_at IS NULL" params = [xform.pk] + where_params else: if json_only: - records = records.values_list('json', flat=True) + records = records.values_list("json", flat=True) if query and isinstance(query, list): for qry in query: - w, wp = get_where_clause(qry, known_integers) - records = records.extra(where=w, params=wp) + _where, _where_params = get_where_clause(qry, known_integers) + records = records.extra(where=_where, params=_where_params) else: if where_params: @@ -216,18 +269,21 @@ def get_sql_with_params(xform, query=None, fields=None, sort=None, start=None, # apply sorting if not count and sort: + # pylint: disable=protected-access if ParsedInstance._has_json_fields(sort): if not fields: # we have to do a sql query for json field order sql, params = records.query.sql_with_params() params = list(params) + json_order_by_params( - sort, none_json_fields=NONE_JSON_FIELDS) - sql = u"%s %s" % (sql, json_order_by( - sort, - none_json_fields=NONE_JSON_FIELDS, - model_name="logger_instance")) - elif not fields: - records = records.order_by(*sort) + sort, none_json_fields=NONE_JSON_FIELDS + ) + _json_order_by = json_order_by( + sort, none_json_fields=NONE_JSON_FIELDS, model_name="logger_instance" + ) + sql = f"{sql} {_json_order_by}" + else: + if not fields: + records = records.order_by(*sort) records, sql, params = _start_index_limit( records, sql, fields, params, sort, start_index, limit @@ -236,36 +292,61 @@ def get_sql_with_params(xform, query=None, fields=None, sort=None, start=None, return sql, params, records -def query_data(xform, query=None, fields=None, sort=None, start=None, - end=None, start_index=None, limit=None, count=None, - json_only: bool = True): +def query_data( + xform, + query=None, + fields=None, + sort=None, + start=None, + end=None, + start_index=None, + limit=None, + count=None, + json_only: bool = True, +): + """Query the submissions table and returns the results.""" sql, params, records = get_sql_with_params( - xform, query, fields, sort, start, end, start_index, limit, count, - json_only=json_only + xform, + query, + fields, + sort, + start, + end, + start_index, + limit, + count, + json_only=json_only, ) if fields and isinstance(fields, six.string_types): fields = json.loads(fields) sort = _get_sort_fields(sort) + # pylint: disable=protected-access if (ParsedInstance._has_json_fields(sort) or fields) and sql: records = _query_iterator(sql, fields, params, count) if count and isinstance(records, types.GeneratorType): - return [i for i in records] - elif count: + return list(records) + if count: return [{"count": records.count()}] return records class ParsedInstance(models.Model): - USERFORM_ID = u'_userform_id' - STATUS = u'_status' + """ + ParsedInstance - parsed XML submission, represents the XML submissions as a python + object. + """ + + USERFORM_ID = "_userform_id" + STATUS = "_status" DEFAULT_LIMIT = settings.PARSED_INSTANCE_DEFAULT_LIMIT DEFAULT_BATCHSIZE = settings.PARSED_INSTANCE_DEFAULT_BATCHSIZE instance = models.OneToOneField( - Instance, related_name="parsed_instance", on_delete=models.CASCADE) + Instance, related_name="parsed_instance", on_delete=models.CASCADE + ) start_time = models.DateTimeField(null=True) end_time = models.DateTimeField(null=True) # TODO: decide if decimal field is better than float field. @@ -282,64 +363,70 @@ def _has_json_fields(cls, sort_list): """ fields = [f.name for f in Instance._meta.get_fields()] - return any([i for i in sort_list if i.lstrip('-') not in fields]) + return any(i for i in sort_list if i.lstrip("-") not in fields) def to_dict_for_mongo(self): - d = self.to_dict() + """ + Return the XForm XML submission as a python object. + """ + data_dict = self.to_dict() data = { UUID: self.instance.uuid, ID: self.instance.id, BAMBOO_DATASET_ID: self.instance.xform.bamboo_dataset, - self.USERFORM_ID: u'%s_%s' % ( - self.instance.xform.user.username, - self.instance.xform.id_string), + self.USERFORM_ID: ( + f"{self.instance.xform.user.username}_" + f"{self.instance.xform.id_string}" + ), ATTACHMENTS: _get_attachments_from_instance(self.instance), self.STATUS: self.instance.status, GEOLOCATION: [self.lat, self.lng], - SUBMISSION_TIME: self.instance.date_created.strftime( - MONGO_STRFTIME), - DATE_MODIFIED: self.instance.date_modified.strftime( - MONGO_STRFTIME), + SUBMISSION_TIME: self.instance.date_created.strftime(MONGO_STRFTIME), + DATE_MODIFIED: self.instance.date_modified.strftime(MONGO_STRFTIME), TAGS: list(self.instance.tags.names()), NOTES: self.get_notes(), - SUBMITTED_BY: self.instance.user.username - if self.instance.user else None, + SUBMITTED_BY: self.instance.user.username if self.instance.user else None, VERSION: self.instance.version, DURATION: self.instance.get_duration(), XFORM_ID: self.instance.xform.pk, TOTAL_MEDIA: self.instance.total_media, MEDIA_COUNT: self.instance.media_count, - MEDIA_ALL_RECEIVED: self.instance.media_all_received + MEDIA_ALL_RECEIVED: self.instance.media_all_received, } if isinstance(self.instance.deleted_at, datetime.datetime): data[DELETEDAT] = self.instance.deleted_at.strftime(MONGO_STRFTIME) if self.instance.has_a_review: - review = self.get_latest_review() + review = self.instance.get_latest_review() if review: data[REVIEW_STATUS] = review.status - data[REVIEW_DATE] = review.date_created.strftime( - MONGO_STRFTIME) + data[REVIEW_DATE] = review.date_created.strftime(MONGO_STRFTIME) if review.get_note_text(): data[REVIEW_COMMENT] = review.get_note_text() - data[EDITED] = (True if self.instance.submission_history.count() > 0 - else False) + data[EDITED] = self.instance.submission_history.count() > 0 - d.update(data) + data_dict.update(data) - return dict_for_mongo(d) + return dict_for_mongo(data_dict) def to_dict(self): + """ + Returns a python dictionary object of a submission. + """ if not hasattr(self, "_dict_cache"): + # pylint: disable=attribute-defined-outside-init self._dict_cache = self.instance.get_dict() return self._dict_cache @classmethod def dicts(cls, xform): - qs = cls.objects.filter(instance__xform=xform) - for parsed_instance in queryset_iterator(qs): + """ + Iterates over a forms submissions. + """ + queryset = cls.objects.filter(instance__xform=xform) + for parsed_instance in queryset_iterator(queryset): yield parsed_instance.to_dict() def _get_name_for_type(self, type_value): @@ -351,10 +438,11 @@ def _get_name_for_type(self, type_value): representation what the 'name' was for a given type_value ('start' or 'end') """ - datadict = json.loads(self.instance.xform.json) - for item in datadict['children']: - if isinstance(item, dict) and item.get(u'type') == type_value: - return item['name'] + datadict = self.instance.xform.json_dict() + for item in datadict["children"]: + if isinstance(item, dict) and item.get("type") == type_value: + return item["name"] + return None # TODO: figure out how much of this code should be here versus # data_dictionary.py. @@ -369,24 +457,29 @@ def save(self, *args, **kwargs): # noqa self.start_time = None self.end_time = None self._set_geopoint() - super(ParsedInstance, self).save(*args, **kwargs) # noqa + super().save(*args, **kwargs) # noqa def add_note(self, note): + """ + Add a note for the instance. + """ note = Note(instance=self.instance, note=note) note.save() - def remove_note(self, pk): - note = self.instance.notes.get(pk=pk) + def remove_note(self, note_id): + """ + Deletes the note with the `pk` as ``note_id`` + """ + note = self.instance.notes.get(pk=note_id) note.delete() def get_notes(self): notes = [] note_qs = self.instance.notes.values( - 'id', 'note', 'date_created', 'date_modified') + "id", "note", "date_created", "date_modified" + ) for note in note_qs: - note['date_created'] = note['date_created'].strftime( - MONGO_STRFTIME) - note['date_modified'] = note['date_modified'].strftime( - MONGO_STRFTIME) + note["date_created"] = note["date_created"].strftime(MONGO_STRFTIME) + note["date_modified"] = note["date_modified"].strftime(MONGO_STRFTIME) notes.append(note) return notes diff --git a/onadata/apps/viewer/parsed_instance_tools.py b/onadata/apps/viewer/parsed_instance_tools.py index 9f9c179f7e..eca1828f5a 100644 --- a/onadata/apps/viewer/parsed_instance_tools.py +++ b/onadata/apps/viewer/parsed_instance_tools.py @@ -1,73 +1,70 @@ -import json -import six +# -*- coding: utf-8 -*- +""" +ParsedInstance model utility functions +""" import datetime +import json from builtins import str as text -from future.utils import iteritems from typing import Any, Tuple -from onadata.libs.utils.common_tags import MONGO_STRFTIME, DATE_FORMAT +import six + +from onadata.libs.utils.common_tags import DATE_FORMAT, MONGO_STRFTIME -KNOWN_DATES = ['_submission_time'] +KNOWN_DATES = ["_submission_time"] NONE_JSON_FIELDS = { - '_submission_time': 'date_created', - '_date_modified': 'date_modified', - '_id': 'id', - '_version': 'version', - '_last_edited': 'last_edited' + "_submission_time": "date_created", + "_date_modified": "date_modified", + "_id": "id", + "_version": "version", + "_last_edited": "last_edited", } +OPERANDS = {"$gt": ">", "$gte": ">=", "$lt": "<", "$lte": "<=", "$i": "~*"} -def _json_sql_str(key, known_integers=None, known_dates=None, - known_decimals=None): +def _json_sql_str(key, known_integers=None, known_dates=None, known_decimals=None): if known_integers is None: known_integers = [] if known_dates is None: known_dates = [] if known_decimals is None: known_decimals = [] - _json_str = u"json->>%s" + _json_str = "json->>%s" if key in known_integers: - _json_str = u"CAST(json->>%s AS INT)" + _json_str = "CAST(json->>%s AS INT)" elif key in known_dates: - _json_str = u"CAST(json->>%s AS TIMESTAMP)" + _json_str = "CAST(json->>%s AS TIMESTAMP)" elif key in known_decimals: - _json_str = u"CAST(json->>%s AS DECIMAL)" + _json_str = "CAST(json->>%s AS DECIMAL)" return _json_str +# pylint: disable=too-many-locals,too-many-branches def _parse_where(query, known_integers, known_decimals, or_where, or_params): # using a dictionary here just incase we will need to filter using # other table columns where, where_params = [], [] - OPERANDS = { - '$gt': '>', - '$gte': '>=', - '$lt': '<', - '$lte': '<=', - '$i': '~*' - } - for (field_key, field_value) in iteritems(query): + # pylint: disable=too-many-nested-blocks + for (field_key, field_value) in six.iteritems(query): if isinstance(field_value, dict): if field_key in NONE_JSON_FIELDS: json_str = NONE_JSON_FIELDS.get(field_key) else: json_str = _json_sql_str( - field_key, known_integers, KNOWN_DATES, known_decimals) - for (key, value) in iteritems(field_value): + field_key, known_integers, KNOWN_DATES, known_decimals + ) + for (key, value) in six.iteritems(field_value): _v = None if key in OPERANDS: - where.append( - u' '.join([json_str, OPERANDS.get(key), u'%s']) - ) + where.append(" ".join([json_str, OPERANDS.get(key), "%s"])) _v = value if field_key in KNOWN_DATES: raw_date = value for date_format in (MONGO_STRFTIME, DATE_FORMAT): try: - _v = datetime.datetime.strptime(raw_date[:19], - date_format) + _v = datetime.datetime.strptime(raw_date[:19], date_format) except ValueError: pass if field_key in NONE_JSON_FIELDS: @@ -76,12 +73,12 @@ def _parse_where(query, known_integers, known_decimals, or_where, or_params): where_params.extend((field_key, text(_v))) else: if field_key in NONE_JSON_FIELDS: - where.append("{} = %s".format(NONE_JSON_FIELDS[field_key])) + where.append(f"{NONE_JSON_FIELDS[field_key]} = %s") where_params.extend([text(field_value)]) elif field_value is None: where.append(f"json->>'{field_key}' IS NULL") else: - where.append(u"json->>%s = %s") + where.append("json->>%s = %s") where_params.extend((field_key, text(field_value))) return where + or_where, where_params + or_params @@ -102,55 +99,58 @@ def _merge_duplicate_keys(pairs: Tuple[str, Any]): return ret -def get_where_clause(query, form_integer_fields=None, - form_decimal_fields=None): +def get_where_clause(query, form_integer_fields=None, form_decimal_fields=None): + """ + Returns where clause and related parameters. + """ if form_integer_fields is None: form_integer_fields = [] if form_decimal_fields is None: form_decimal_fields = [] - known_integers = ['_id'] + form_integer_fields + known_integers = ["_id"] + form_integer_fields known_decimals = form_decimal_fields where = [] where_params = [] + # pylint: disable=too-many-nested-blocks try: if query and isinstance(query, (dict, six.string_types)): - query = query if isinstance(query, dict) else json.loads( - query, object_pairs_hook=_merge_duplicate_keys) + query = ( + query + if isinstance(query, dict) + else json.loads(query, object_pairs_hook=_merge_duplicate_keys) + ) or_where = [] or_params = [] if isinstance(query, list): query = query[0] - if isinstance(query, dict) and '$or' in list(query): - or_dict = query.pop('$or') + if isinstance(query, dict) and "$or" in list(query): + or_dict = query.pop("$or") for or_query in or_dict: for k, v in or_query.items(): if v is None: - or_where.extend( - [u"json->>'{}' IS NULL".format(k)]) + or_where.extend([f"json->>'{k}' IS NULL"]) elif isinstance(v, list): for value in v: or_where.extend(["json->>%s = %s"]) or_params.extend([k, value]) else: - or_where.extend( - [u"json->>%s = %s"]) + or_where.extend(["json->>%s = %s"]) or_params.extend([k, v]) - or_where = [u"".join([u"(", u" OR ".join(or_where), u")"])] + or_where = ["".join(["(", " OR ".join(or_where), ")"])] - where, where_params = _parse_where(query, known_integers, - known_decimals, or_where, - or_params) + where, where_params = _parse_where( + query, known_integers, known_decimals, or_where, or_params + ) except (ValueError, AttributeError) as e: - if query and isinstance(query, six.string_types) and \ - query.startswith('{'): + if query and isinstance(query, six.string_types) and query.startswith("{"): raise e # cast query param to text - where = [u"json::text ~* cast(%s as text)"] + where = ["json::text ~* cast(%s as text)"] where_params = [query] return where, where_params diff --git a/onadata/apps/viewer/tasks.py b/onadata/apps/viewer/tasks.py index 54a846e38c..1126caa81d 100644 --- a/onadata/apps/viewer/tasks.py +++ b/onadata/apps/viewer/tasks.py @@ -5,7 +5,7 @@ import sys from datetime import timedelta -from future.utils import iteritems +from six import iteritems from django.conf import settings from django.shortcuts import get_object_or_404 @@ -17,21 +17,23 @@ from onadata.apps.viewer.models.export import Export, ExportTypeError from onadata.libs.exceptions import NoRecordsFoundError from onadata.libs.utils.common_tools import get_boolean_value, report_exception -from onadata.libs.utils.export_tools import (generate_attachments_zip_export, - generate_export, - generate_external_export, - generate_kml_export, - generate_osm_export) +from onadata.libs.utils.export_tools import ( + generate_attachments_zip_export, + generate_export, + generate_external_export, + generate_kml_export, + generate_osm_export, +) from onadata.celery import app -EXPORT_QUERY_KEY = 'query' +EXPORT_QUERY_KEY = "query" def _get_export_object(export_id): try: return Export.objects.get(id=export_id) except Export.DoesNotExist: - if getattr(settings, 'SLAVE_DATABASES', []): + if getattr(settings, "SLAVE_DATABASES", []): from multidb.pinning import use_master with use_master: @@ -41,11 +43,7 @@ def _get_export_object(export_id): def _get_export_details(username, id_string, export_id): - details = { - 'export_id': export_id, - 'username': username, - 'id_string': id_string - } + details = {"export_id": export_id, "username": username, "id_string": id_string} return details @@ -65,24 +63,27 @@ def _create_export(xform, export_type, options): for (key, value) in iteritems(options) if key in Export.EXPORT_OPTION_FIELDS } - if query and 'query' not in export_options: - export_options['query'] = query + if query and "query" not in export_options: + export_options["query"] = query return Export.objects.create( - xform=xform, export_type=export_type, options=export_options) + xform=xform, export_type=export_type, options=export_options + ) export = _create_export(xform, export_type, options) result = None export_id = export.id - options.update({ - 'username': username, - 'id_string': id_string, - 'export_id': export_id, - 'query': query, - 'force_xlsx': force_xlsx - }) + options.update( + { + "username": username, + "id_string": id_string, + "export_id": export_id, + "query": query, + "force_xlsx": force_xlsx, + } + ) export_types = { Export.XLS_EXPORT: create_xls_export, @@ -93,7 +94,7 @@ def _create_export(xform, export_type, options): Export.ZIP_EXPORT: create_zip_export, Export.KML_EXPORT: create_kml_export, Export.OSM_EXPORT: create_osm_export, - Export.EXTERNAL_EXPORT: create_external_export + Export.EXTERNAL_EXPORT: create_external_export, } # start async export @@ -113,7 +114,7 @@ def _create_export(xform, export_type, options): # when celery is running eager, the export has been generated by the # time we get here so lets retrieve the export object a fresh before we # save - if getattr(settings, 'CELERY_TASK_ALWAYS_EAGER', False): + if getattr(settings, "CELERY_TASK_ALWAYS_EAGER", False): export = get_object_or_404(Export, id=export.id) export.task_id = result.task_id export.save() @@ -129,7 +130,7 @@ def create_xls_export(username, id_string, export_id, **options): # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state force_xlsx = options.get("force_xlsx", True) - options["extension"] = 'xlsx' if force_xlsx else 'xls' + options["extension"] = "xlsx" if force_xlsx else "xls" try: export = _get_export_object(export_id) @@ -140,8 +141,9 @@ def create_xls_export(username, id_string, export_id, **options): # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery try: - gen_export = generate_export(Export.XLS_EXPORT, export.xform, - export_id, options) + gen_export = generate_export( + Export.XLS_EXPORT, export.xform, export_id, options + ) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.save() @@ -150,8 +152,10 @@ def create_xls_export(username, id_string, export_id, **options): report_exception( "XLS Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) # Raise for now to let celery know we failed # - doesnt seem to break celery` raise @@ -171,8 +175,9 @@ def create_csv_export(username, id_string, export_id, **options): try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery - gen_export = generate_export(Export.CSV_EXPORT, export.xform, - export_id, options) + gen_export = generate_export( + Export.CSV_EXPORT, export.xform, export_id, options + ) except NoRecordsFoundError: # not much we can do but we don't want to report this as the user # should not even be on this page if the survey has no records @@ -187,8 +192,10 @@ def create_csv_export(username, id_string, export_id, **options): report_exception( "CSV Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -212,7 +219,8 @@ def create_kml_export(username, id_string, export_id, **options): id_string, export_id, options, - xform=export.xform) + xform=export.xform, + ) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.save() @@ -220,8 +228,10 @@ def create_kml_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "KML Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -245,7 +255,8 @@ def create_osm_export(username, id_string, export_id, **options): id_string, export_id, options, - xform=export.xform) + xform=export.xform, + ) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.error_message = str(e) @@ -254,8 +265,10 @@ def create_osm_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "OSM Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -274,7 +287,8 @@ def create_zip_export(username, id_string, export_id, **options): id_string, export_id, options, - xform=export.xform) + xform=export.xform, + ) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.error_message = str(e) @@ -283,13 +297,17 @@ def create_zip_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "Zip Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + ) raise else: if not settings.TESTING_MODE: delete_export.apply_async( - (), {'export_id': gen_export.id}, - countdown=settings.ZIP_EXPORT_COUNTDOWN) + (), + {"export_id": gen_export.id}, + countdown=settings.ZIP_EXPORT_COUNTDOWN, + ) return gen_export.id @@ -303,8 +321,9 @@ def create_csv_zip_export(username, id_string, export_id, **options): try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery - gen_export = generate_export(Export.CSV_ZIP_EXPORT, export.xform, - export_id, options) + gen_export = generate_export( + Export.CSV_ZIP_EXPORT, export.xform, export_id, options + ) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.error_message = str(e) @@ -313,8 +332,10 @@ def create_csv_zip_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "CSV ZIP Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -330,8 +351,9 @@ def create_sav_zip_export(username, id_string, export_id, **options): try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery - gen_export = generate_export(Export.SAV_ZIP_EXPORT, export.xform, - export_id, options) + gen_export = generate_export( + Export.SAV_ZIP_EXPORT, export.xform, export_id, options + ) except (Exception, NoRecordsFoundError, TypeError) as e: export.internal_status = Export.FAILED export.save() @@ -339,8 +361,10 @@ def create_sav_zip_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "SAV ZIP Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -361,7 +385,8 @@ def create_external_export(username, id_string, export_id, **options): id_string, export_id, options, - xform=export.xform) + xform=export.xform, + ) except (Exception, NoRecordsFoundError, ConnectionError) as e: export.internal_status = Export.FAILED export.save() @@ -369,8 +394,10 @@ def create_external_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "External Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -387,8 +414,9 @@ def create_google_sheet_export(username, id_string, export_id, **options): export = _get_export_object(export_id) # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery - gen_export = generate_export(Export.GOOGLE_SHEETS_EXPORT, export.xform, - export_id, options) + gen_export = generate_export( + Export.GOOGLE_SHEETS_EXPORT, export.xform, export_id, options + ) except (Exception, NoRecordsFoundError, ConnectionError) as e: export.internal_status = Export.FAILED export.save() @@ -396,8 +424,10 @@ def create_google_sheet_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "Google Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -427,7 +457,8 @@ def mark_expired_pending_exports_as_failed(): # pylint: disable=invalid-name task_lifespan = settings.EXPORT_TASK_LIFESPAN time_threshold = timezone.now() - timedelta(hours=task_lifespan) exports = Export.objects.filter( - internal_status=Export.PENDING, created_on__lt=time_threshold) + internal_status=Export.PENDING, created_on__lt=time_threshold + ) exports.update(internal_status=Export.FAILED) @@ -439,5 +470,6 @@ def delete_expired_failed_exports(): task_lifespan = settings.EXPORT_TASK_LIFESPAN time_threshold = timezone.now() - timedelta(hours=task_lifespan) exports = Export.objects.filter( - internal_status=Export.FAILED, created_on__lt=time_threshold) + internal_status=Export.FAILED, created_on__lt=time_threshold + ) exports.delete() diff --git a/onadata/apps/viewer/tests/test_export_list.py b/onadata/apps/viewer/tests/test_export_list.py index 468a75ecac..16cf0a8721 100644 --- a/onadata/apps/viewer/tests/test_export_list.py +++ b/onadata/apps/viewer/tests/test_export_list.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Test export list view. +""" import os from django.conf import settings @@ -10,20 +14,31 @@ class TestExportList(TestBase): + """ + Test export list view. + """ def setUp(self): - super(TestExportList, self).setUp() + super().setUp() self._publish_transportation_form() survey = self.surveys[0] self._make_submission( os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'instances', survey, survey + '.xml')) + self.this_directory, + "fixtures", + "transportation", + "instances", + survey, + survey + ".xml", + ) + ) def test_unauthorised_users_cannot_export_form_data(self): - kwargs = {'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': Export.CSV_EXPORT} + kwargs = { + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": Export.CSV_EXPORT, + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) @@ -33,43 +48,54 @@ def test_unauthorised_users_cannot_export_form_data(self): '', - response.content.decode('utf-8')) + response.content.decode("utf-8"), + ) self.assertEqual(response.status_code, 200) def test_unsupported_type_export(self): - kwargs = {'username': self.user.username.upper(), - 'id_string': self.xform.id_string.upper(), - 'export_type': 'gdoc'} + kwargs = { + "username": self.user.username.upper(), + "id_string": self.xform.id_string.upper(), + "export_type": "gdoc", + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) self.assertEqual(response.status_code, 400) def test_export_data_with_unavailable_id_string(self): - kwargs = {'username': self.user.username.upper(), - 'id_string': 'random_id_string', - 'export_type': Export.CSV_EXPORT} + kwargs = { + "username": self.user.username.upper(), + "id_string": "random_id_string", + "export_type": Export.CSV_EXPORT, + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) self.assertEqual(response.status_code, 404) - kwargs = {'username': self.user.username.upper(), - 'id_string': 'random_id_string', - 'export_type': Export.ZIP_EXPORT} + kwargs = { + "username": self.user.username.upper(), + "id_string": "random_id_string", + "export_type": Export.ZIP_EXPORT, + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) self.assertEqual(response.status_code, 404) - kwargs = {'username': self.user.username.upper(), - 'id_string': 'random_id_string', - 'export_type': Export.KML_EXPORT} + kwargs = { + "username": self.user.username.upper(), + "id_string": "random_id_string", + "export_type": Export.KML_EXPORT, + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) self.assertEqual(response.status_code, 404) def test_csv_export_list(self): - kwargs = {'username': self.user.username.upper(), - 'id_string': self.xform.id_string.upper(), - 'export_type': Export.CSV_EXPORT} + kwargs = { + "username": self.user.username.upper(), + "id_string": self.xform.id_string.upper(), + "export_type": Export.CSV_EXPORT, + } # test csv url = reverse(export_list, kwargs=kwargs) @@ -77,173 +103,209 @@ def test_csv_export_list(self): self.assertEqual(response.status_code, 200) def test_xls_export_list(self): - kwargs = {'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': Export.XLS_EXPORT} + kwargs = { + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": Export.XLS_EXPORT, + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) self.assertEqual(response.status_code, 200) def test_kml_export_list(self): - kwargs = {'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': Export.KML_EXPORT} + kwargs = { + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": Export.KML_EXPORT, + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) self.assertEqual(response.status_code, 200) def test_zip_export_list(self): - kwargs = {'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': Export.ZIP_EXPORT} + kwargs = { + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": Export.ZIP_EXPORT, + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) self.assertEqual(response.status_code, 200) def test_csv_zip_export_list(self): - kwargs = {'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': Export.CSV_ZIP_EXPORT} + kwargs = { + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": Export.CSV_ZIP_EXPORT, + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) self.assertEqual(response.status_code, 200) def test_sav_zip_export_list(self): - kwargs = {'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': Export.SAV_ZIP_EXPORT} + kwargs = { + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": Export.SAV_ZIP_EXPORT, + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) self.assertEqual(response.status_code, 200) def test_external_export_list(self): - kwargs = {'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': Export.EXTERNAL_EXPORT} - server = 'http://localhost:8080/xls/23fa4c38c0054748a984ffd89021a295' - data_value = 'template 1 |{0}'.format(server) + kwargs = { + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": Export.EXTERNAL_EXPORT, + } + server = "http://localhost:8080/xls/23fa4c38c0054748a984ffd89021a295" + data_value = "template 1 |{0}".format(server) meta = MetaData.external_export(self.xform, data_value) custom_params = { - 'meta': meta.id, + "meta": meta.id, } url = reverse(export_list, kwargs=kwargs) count = len(Export.objects.all()) response = self.client.get(url, custom_params) self.assertEqual(response.status_code, 200) count1 = len(Export.objects.all()) - self.assertEquals(count + 1, count1) + self.assertEqual(count + 1, count1) def test_external_export_list_no_template(self): - kwargs = {'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': Export.EXTERNAL_EXPORT} + kwargs = { + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": Export.EXTERNAL_EXPORT, + } url = reverse(export_list, kwargs=kwargs) count = len(Export.objects.all()) response = self.client.get(url) self.assertEqual(response.status_code, 403) - self.assertEquals(response.content.decode('utf-8'), - u'No XLS Template set.') + self.assertEqual(response.content.decode("utf-8"), "No XLS Template set.") count1 = len(Export.objects.all()) - self.assertEquals(count, count1) + self.assertEqual(count, count1) class TestDataExportURL(TestBase): - def setUp(self): super(TestDataExportURL, self).setUp() self._publish_transportation_form() def _filename_from_disposition(self, content_disposition): - filename_pos = content_disposition.index('filename=') + filename_pos = content_disposition.index("filename=") self.assertTrue(filename_pos != -1) - return content_disposition[filename_pos + len('filename='):] + return content_disposition[filename_pos + len("filename=") :] def test_csv_export_url(self): self._submit_transport_instance() - url = reverse('csv_export', kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - }) + url = reverse( + "csv_export", + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ) response = self.client.get(url) headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], 'application/csv') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/csv") + content_disposition = headers["Content-Disposition"] filename = self._filename_from_disposition(content_disposition) basename, ext = os.path.splitext(filename) - self.assertEqual(ext, '.csv') + self.assertEqual(ext, ".csv") def test_csv_export_url_without_records(self): # this has been refactored so that if NoRecordsFound Exception is # thrown, it will return an empty csv containing only the xform schema - url = reverse('csv_export', kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - }) + url = reverse( + "csv_export", + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ) response = self.client.get(url) self.assertEqual(response.status_code, 200) # Unpack response streaming data - export_data = [i.decode( - 'utf-8').replace('\n', '').split( - ',') for i in response.streaming_content] + export_data = [ + i.decode("utf-8").replace("\n", "").split(",") + for i in response.streaming_content + ] xform_headers = self.xform.get_headers() # Remove review headers from xform headers - for x in ['_review_status', '_review_comment']: + for x in ["_review_status", "_review_comment"]: xform_headers.remove(x) # Test export data returned is xform headers list self.assertEqual(xform_headers, export_data[0]) def test_xls_export_url(self): self._submit_transport_instance() - url = reverse('xls_export', kwargs={ - 'username': self.user.username.upper(), - 'id_string': self.xform.id_string.upper(), - }) + url = reverse( + "xls_export", + kwargs={ + "username": self.user.username.upper(), + "id_string": self.xform.id_string.upper(), + }, + ) response = self.client.get(url) headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], - 'application/vnd.openxmlformats') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/vnd.openxmlformats") + content_disposition = headers["Content-Disposition"] filename = self._filename_from_disposition(content_disposition) basename, ext = os.path.splitext(filename) - self.assertEqual(ext, '.xlsx') + self.assertEqual(ext, ".xlsx") def test_csv_zip_export_url(self): self._submit_transport_instance() - url = reverse('csv_zip_export', kwargs={ - 'username': self.user.username.upper(), - 'id_string': self.xform.id_string.upper(), - }) + url = reverse( + "csv_zip_export", + kwargs={ + "username": self.user.username.upper(), + "id_string": self.xform.id_string.upper(), + }, + ) response = self.client.get(url) headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], 'application/zip') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/zip") + content_disposition = headers["Content-Disposition"] filename = self._filename_from_disposition(content_disposition) basename, ext = os.path.splitext(filename) - self.assertEqual(ext, '.zip') + self.assertEqual(ext, ".zip") def test_sav_zip_export_url(self): - filename = os.path.join(settings.PROJECT_ROOT, 'apps', 'logger', - 'tests', 'fixtures', 'childrens_survey.xlsx') + filename = os.path.join( + settings.PROJECT_ROOT, + "apps", + "logger", + "tests", + "fixtures", + "childrens_survey.xlsx", + ) self._publish_xls_file_and_set_xform(filename) - url = reverse('sav_zip_export', kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - }) + url = reverse( + "sav_zip_export", + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ) response = self.client.get(url) headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], 'application/zip') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/zip") + content_disposition = headers["Content-Disposition"] filename = self._filename_from_disposition(content_disposition) basename, ext = os.path.splitext(filename) - self.assertEqual(ext, '.zip') + self.assertEqual(ext, ".zip") def test_sav_zip_export_long_variable_length(self): self._submit_transport_instance() - url = reverse('sav_zip_export', kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - }) + url = reverse( + "sav_zip_export", + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ) response = self.client.get(url) self.assertEqual(response.status_code, 200) diff --git a/onadata/apps/viewer/tests/test_exports.py b/onadata/apps/viewer/tests/test_exports.py index d9a5ece118..4d3846bfcf 100644 --- a/onadata/apps/viewer/tests/test_exports.py +++ b/onadata/apps/viewer/tests/test_exports.py @@ -1336,7 +1336,7 @@ def test_create_external_export_without_template(self): response = self.client.post(create_export_url) self.assertEqual(response.status_code, 403) - self.assertEquals(response.content, b'No XLS Template set.') + self.assertEqual(response.content, b'No XLS Template set.') self.assertEqual(Export.objects.count(), num_exports) def test_all_keys_cleaned_of_slashes(self): @@ -1466,4 +1466,4 @@ def test_all_keys_cleaned_of_slashes(self): } result_data = clean_keys_of_slashes(data) - self.assertEquals(expected_data, result_data) + self.assertEqual(expected_data, result_data) diff --git a/onadata/apps/viewer/tests/test_kml_export.py b/onadata/apps/viewer/tests/test_kml_export.py index c5573f5054..375b369568 100644 --- a/onadata/apps/viewer/tests/test_kml_export.py +++ b/onadata/apps/viewer/tests/test_kml_export.py @@ -1,7 +1,7 @@ import os from django.urls import reverse -from future.utils import iteritems +from six import iteritems from onadata.apps.logger.models.instance import Instance from onadata.apps.main.tests.test_base import TestBase @@ -9,43 +9,46 @@ class TestKMLExport(TestBase): - def _publish_survey(self): self.this_directory = os.path.dirname(__file__) xls_path = self._fixture_path("gps", "gps.xlsx") TestBase._publish_xls_file(self, xls_path) def test_kml_export(self): - id_string = 'gps' + id_string = "gps" self._publish_survey() self._make_submissions_gps() - self.fixtures = os.path.join( - self.this_directory, 'fixtures', 'kml_export') + self.fixtures = os.path.join(self.this_directory, "fixtures", "kml_export") url = reverse( - kml_export, - kwargs={'username': self.user.username, 'id_string': id_string}) + kml_export, kwargs={"username": self.user.username, "id_string": id_string} + ) response = self.client.get(url) instances = Instance.objects.filter( - xform__user=self.user, xform__id_string=id_string, - geom__isnull=False - ).order_by('id') + xform__user=self.user, xform__id_string=id_string, geom__isnull=False + ).order_by("id") self.assertEqual(instances.count(), 2) # create a tuple of replacement data per instance - replacement_data = [["{:,}".format(x) for x in [ - i.pk, i.point.x, i.point.y]] for i in instances] + replacement_data = [ + ["{:,}".format(x) for x in [i.pk, i.point.x, i.point.y]] for i in instances + ] # assuming 2 instances, flatten and assign to template names - replacement_dict = dict(zip(['pk1', 'x1', 'y1', 'pk2', 'x2', 'y2'], - [i for s in replacement_data for i in s])) - - with open(os.path.join(self.fixtures, 'export.kml')) as f: + replacement_dict = dict( + zip( + ["pk1", "x1", "y1", "pk2", "x2", "y2"], + [i for s in replacement_data for i in s], + ) + ) + + with open(os.path.join(self.fixtures, "export.kml")) as f: expected_content = f.read() for (template_name, template_data) in iteritems(replacement_dict): expected_content = expected_content.replace( - '{{%s}}' % template_name, template_data) + "{{%s}}" % template_name, template_data + ) self.assertMultiLineEqual( - expected_content.strip(), - response.content.decode('utf-8').strip()) + expected_content.strip(), response.content.decode("utf-8").strip() + ) diff --git a/onadata/apps/viewer/tests/test_tasks.py b/onadata/apps/viewer/tests/test_tasks.py index a080de361e..dec4631144 100644 --- a/onadata/apps/viewer/tests/test_tasks.py +++ b/onadata/apps/viewer/tests/test_tasks.py @@ -42,7 +42,7 @@ def test_create_async(self): export = result[0] self.assertTrue(export.id) self.assertIn("username", options) - self.assertEquals(options.get("id_string"), self.xform.id_string) + self.assertEqual(options.get("id_string"), self.xform.id_string) def test_mark_expired_pending_exports_as_failed(self): self._publish_transportation_form_and_submit_instance() @@ -56,7 +56,7 @@ def test_mark_expired_pending_exports_as_failed(self): export.save() mark_expired_pending_exports_as_failed() export = Export.objects.filter(pk=export.pk).first() - self.assertEquals(export.internal_status, Export.FAILED) + self.assertEqual(export.internal_status, Export.FAILED) def test_delete_expired_failed_exports(self): self._publish_transportation_form_and_submit_instance() @@ -70,4 +70,4 @@ def test_delete_expired_failed_exports(self): export.save() pk = export.pk delete_expired_failed_exports() - self.assertEquals(Export.objects.filter(pk=pk).first(), None) + self.assertEqual(Export.objects.filter(pk=pk).first(), None) diff --git a/onadata/apps/viewer/views.py b/onadata/apps/viewer/views.py index 03e24606a4..7373fb23ea 100644 --- a/onadata/apps/viewer/views.py +++ b/onadata/apps/viewer/views.py @@ -1,34 +1,41 @@ # -*- coding: utf-8 -*- +# pylint: disable=too-many-lines """ data views. """ -import json import os from datetime import datetime from tempfile import NamedTemporaryFile from time import strftime, strptime from wsgiref.util import FileWrapper -import pytz -import requests -from dict2xml import dict2xml from django.conf import settings +from django.contrib.auth import get_user_model from django.contrib.auth.decorators import login_required -from django.contrib.auth.models import User from django.core.files.storage import FileSystemStorage, get_storage_class -from django.http import (HttpResponse, HttpResponseBadRequest, - HttpResponseForbidden, HttpResponseNotFound, - HttpResponseRedirect) +from django.http import ( + HttpResponse, + HttpResponseBadRequest, + HttpResponseForbidden, + HttpResponseNotFound, + HttpResponseRedirect, + JsonResponse, +) from django.shortcuts import get_object_or_404, redirect, render from django.template import loader from django.urls import reverse -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.views.decorators.http import require_POST + +import pytz +import requests +from dict2xml import dict2xml from dpath import util as dpath_util -from oauth2client import client as google_client -from oauth2client.contrib.django_util.storage import \ - DjangoORMStorage as Storage -from savReaderWriter import SPSSIOError + +try: + from savReaderWriter import SPSSIOError +except ImportError: + SPSSIOError = Exception from onadata.apps.logger.models import Attachment from onadata.apps.logger.views import download_jsonform @@ -39,34 +46,53 @@ from onadata.apps.viewer.xls_writer import XlsWriter from onadata.libs.exceptions import NoRecordsFoundError from onadata.libs.utils.chart_tools import build_chart_data +from onadata.libs.utils.common_tools import get_uuid from onadata.libs.utils.export_tools import ( - DEFAULT_GROUP_DELIMITER, generate_export, kml_export_data, - newest_export_for, should_create_new_export, str_to_bool) -from onadata.libs.utils.google import google_flow + DEFAULT_GROUP_DELIMITER, + generate_export, + kml_export_data, + newest_export_for, + should_create_new_export, + str_to_bool, +) +from onadata.libs.utils.google import create_flow from onadata.libs.utils.image_tools import image_url from onadata.libs.utils.log import Actions, audit_log from onadata.libs.utils.logger_tools import ( - generate_content_disposition_header, response_with_mimetype_and_name) -from onadata.libs.utils.user_auth import (get_xform_and_perms, has_permission, - helper_auth_helper) + generate_content_disposition_header, + response_with_mimetype_and_name, +) +from onadata.libs.utils.user_auth import ( + get_xform_and_perms, + has_permission, + helper_auth_helper, +) from onadata.libs.utils.viewer_tools import ( - create_attachments_zipfile, export_def_from_filename, get_form) -from onadata.libs.utils.common_tools import get_uuid + create_attachments_zipfile, + export_def_from_filename, + get_form, +) + +# pylint: disable=invalid-name +User = get_user_model() def _get_start_end_submission_time(request): start = None end = None try: - if request.GET.get('start'): - start = pytz.timezone('UTC').localize( - datetime.strptime(request.GET['start'], '%y_%m_%d_%H_%M_%S')) - if request.GET.get('end'): - end = pytz.timezone('UTC').localize( - datetime.strptime(request.GET['end'], '%y_%m_%d_%H_%M_%S')) + if request.GET.get("start"): + start = pytz.timezone("UTC").localize( + datetime.strptime(request.GET["start"], "%y_%m_%d_%H_%M_%S") + ) + if request.GET.get("end"): + end = pytz.timezone("UTC").localize( + datetime.strptime(request.GET["end"], "%y_%m_%d_%H_%M_%S") + ) except ValueError: return HttpResponseBadRequest( - _("Dates must be in the format YY_MM_DD_hh_mm_ss")) + _("Dates must be in the format YY_MM_DD_hh_mm_ss") + ) return start, end @@ -75,27 +101,27 @@ def encode(time_str): """ Reformat a time string into YYYY-MM-dd HH:mm:ss. """ - return strftime("%Y-%m-%d %H:%M:%S", - strptime(time_str, "%Y_%m_%d_%H_%M_%S")) + return strftime("%Y-%m-%d %H:%M:%S", strptime(time_str, "%Y_%m_%d_%H_%M_%S")) def format_date_for_mongo(time_str): """ Reformat a time string into YYYY-MM-ddTHH:mm:ss. """ - return datetime.strptime(time_str, '%y_%m_%d_%H_%M_%S')\ - .strftime('%Y-%m-%dT%H:%M:%S') + return datetime.strptime(time_str, "%y_%m_%d_%H_%M_%S").strftime( + "%Y-%m-%dT%H:%M:%S" + ) def instances_for_export(data_dictionary, start=None, end=None): """ Returns Instance submission queryset filtered by start and end dates. """ - kwargs = dict() + kwargs = {} if start: - kwargs['date_created__gte'] = start + kwargs["date_created__gte"] = start if end: - kwargs['date_created__lte'] = end + kwargs["date_created__lte"] = end return data_dictionary.instances.filter(**kwargs) @@ -108,11 +134,9 @@ def set_instances_for_export(id_string, owner, request): is successful or not respectively. """ data_dictionary = get_object_or_404( - DataDictionary, - id_string__iexact=id_string, - user=owner, - deletd_at__isnull=True) - start, end = request.GET.get('start'), request.GET.get('end') + DataDictionary, id_string__iexact=id_string, user=owner, deletd_at__isnull=True + ) + start, end = request.GET.get("start"), request.GET.get("end") if start: try: start = encode(start) @@ -121,7 +145,8 @@ def set_instances_for_export(id_string, owner, request): return [ False, HttpResponseBadRequest( - _(u'Start time format must be YY_MM_DD_hh_mm_ss')) + _("Start time format must be YY_MM_DD_hh_mm_ss") + ), ] if end: try: @@ -130,12 +155,12 @@ def set_instances_for_export(id_string, owner, request): # bad format return [ False, - HttpResponseBadRequest( - _(u'End time format must be YY_MM_DD_hh_mm_ss')) + HttpResponseBadRequest(_("End time format must be YY_MM_DD_hh_mm_ss")), ] if start or end: data_dictionary.instances_for_export = instances_for_export( - data_dictionary, start, end) + data_dictionary, start, end + ) return [True, data_dictionary] @@ -147,48 +172,48 @@ def average(values): return sum(values, 0.0) / len(values) if values else None -def map_view(request, username, id_string, template='map.html'): +def map_view(request, username, id_string, template="map.html"): """ Map view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) - data = {'content_user': owner, 'xform': xform} - data['profile'], __ = UserProfile.objects.get_or_create(user=owner) - - data['form_view'] = True - data['jsonform_url'] = reverse( - download_jsonform, - kwargs={"username": username, - "id_string": id_string}) - data['enketo_edit_url'] = reverse( - 'edit_data', - kwargs={"username": username, - "id_string": id_string, - "data_id": 0}) - data['enketo_add_url'] = reverse( - 'enter_data', kwargs={"username": username, - "id_string": id_string}) - - data['enketo_add_with_url'] = reverse( - 'add_submission_with', - kwargs={"username": username, - "id_string": id_string}) - data['mongo_api_url'] = reverse( - 'mongo_view_api', - kwargs={"username": username, - "id_string": id_string}) - data['delete_data_url'] = reverse( - 'delete_data', kwargs={"username": username, - "id_string": id_string}) - data['mapbox_layer'] = MetaData.mapbox_layer_upload(xform) + return HttpResponseForbidden(_("Not shared.")) + data = {"content_user": owner, "xform": xform} + data["profile"], __ = UserProfile.objects.get_or_create(user=owner) + + data["form_view"] = True + data["jsonform_url"] = reverse( + download_jsonform, kwargs={"username": username, "id_string": id_string} + ) + data["enketo_edit_url"] = reverse( + "edit_data", kwargs={"username": username, "id_string": id_string, "data_id": 0} + ) + data["enketo_add_url"] = reverse( + "enter_data", kwargs={"username": username, "id_string": id_string} + ) + + data["enketo_add_with_url"] = reverse( + "add_submission_with", kwargs={"username": username, "id_string": id_string} + ) + data["mongo_api_url"] = reverse( + "mongo_view_api", kwargs={"username": username, "id_string": id_string} + ) + data["delete_data_url"] = reverse( + "delete_data", kwargs={"username": username, "id_string": id_string} + ) + data["mapbox_layer"] = MetaData.mapbox_layer_upload(xform) audit = {"xform": xform.id_string} - audit_log(Actions.FORM_MAP_VIEWED, request.user, owner, - _("Requested map on '%(id_string)s'.") % - {'id_string': xform.id_string}, audit, request) + audit_log( + Actions.FORM_MAP_VIEWED, + request.user, + owner, + _(f"Requested map on '{xform.id_string}'."), + audit, + request, + ) return render(request, template, data) @@ -196,7 +221,7 @@ def map_embed_view(request, username, id_string): """ Embeded map view. """ - return map_view(request, username, id_string, template='map_embed.html') + return map_view(request, username, id_string, template="map_embed.html") def add_submission_with(request, username, id_string): @@ -209,54 +234,55 @@ def geopoint_xpaths(username, id_string): Returns xpaths with elements of type 'geopoint'. """ data_dictionary = DataDictionary.objects.get( - user__username__iexact=username, id_string__iexact=id_string) + user__username__iexact=username, id_string__iexact=id_string + ) return [ e.get_abbreviated_xpath() for e in data_dictionary.get_survey_elements() - if e.bind.get(u'type') == u'geopoint' + if e.bind.get("type") == "geopoint" ] - value = request.GET.get('coordinates') + value = request.GET.get("coordinates") xpaths = geopoint_xpaths(username, id_string) xml_dict = {} for path in xpaths: dpath_util.new(xml_dict, path, value) context = { - 'username': username, - 'id_string': id_string, - 'xml_content': dict2xml(xml_dict) + "username": username, + "id_string": id_string, + "xml_content": dict2xml(xml_dict), } - instance_xml = loader.get_template("instance_add.xml")\ - .render(context) + instance_xml = loader.get_template("instance_add.xml").render(context) url = settings.ENKETO_API_INSTANCE_IFRAME_URL return_url = reverse( - 'thank_you_submission', - kwargs={"username": username, - "id_string": id_string}) + "thank_you_submission", kwargs={"username": username, "id_string": id_string} + ) if settings.DEBUG: - openrosa_url = "https://dev.formhub.org/{}".format(username) + openrosa_url = f"https://dev.formhub.org/{username}" else: - openrosa_url = request.build_absolute_uri("/{}".format(username)) + openrosa_url = request.build_absolute_uri(f"/{username}") payload = { - 'return_url': return_url, - 'form_id': id_string, - 'server_url': openrosa_url, - 'instance': instance_xml, - 'instance_id': get_uuid() + "return_url": return_url, + "form_id": id_string, + "server_url": openrosa_url, + "instance": instance_xml, + "instance_id": get_uuid(), } response = requests.post( url, data=payload, - auth=(settings.ENKETO_API_TOKEN, ''), - verify=getattr(settings, 'VERIFY_SSL', True)) + auth=(settings.ENKETO_API_TOKEN, ""), + verify=getattr(settings, "VERIFY_SSL", True), + ) - return HttpResponse(response.text, content_type='application/json') + # pylint: disable=http-response-with-content-type-json + return HttpResponse(response.text, content_type="application/json") -# pylint: disable=W0613 +# pylint: disable=unused-argument def thank_you_submission(request, username, id_string): """ Thank you view after successful submission. @@ -264,90 +290,100 @@ def thank_you_submission(request, username, id_string): return HttpResponse("Thank You") -# pylint: disable=R0914 -def data_export(request, username, id_string, export_type): +# pylint: disable=too-many-locals +def data_export(request, username, id_string, export_type): # noqa C901 """ Data export view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) + id_string = xform.id_string helper_auth_helper(request) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) query = request.GET.get("query") extension = export_type # check if we should force xlsx - force_xlsx = request.GET.get('xls') != 'true' + force_xlsx = request.GET.get("xls") != "true" if export_type == Export.XLS_EXPORT and force_xlsx: - extension = 'xlsx' + extension = "xlsx" elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]: - extension = 'zip' + extension = "zip" audit = {"xform": xform.id_string, "export_type": export_type} - options = { - "extension": extension, - "username": username, - "id_string": id_string - } + options = {"extension": extension, "username": username, "id_string": id_string} if query: - options['query'] = query + options["query"] = query # check if we need to re-generate, # we always re-generate if a filter is specified - if should_create_new_export(xform, export_type, options) or query or\ - 'start' in request.GET or 'end' in request.GET: + if ( + should_create_new_export(xform, export_type, options) + or query + or "start" in request.GET + or "end" in request.GET + ): # check for start and end params start, end = _get_start_end_submission_time(request) options.update({"start": start, "end": end}) + # pylint: disable=broad-except try: export = generate_export(export_type, xform, None, options) + export_type = export_type.upper() audit_log( - Actions.EXPORT_CREATED, request.user, owner, - _("Created %(export_type)s export on '%(id_string)s'.") % { - 'id_string': xform.id_string, - 'export_type': export_type.upper() - }, audit, request) + Actions.EXPORT_CREATED, + request.user, + owner, + _(f"Created {export_type} export on '{id_string}'."), + audit, + request, + ) except NoRecordsFoundError: return HttpResponseNotFound(_("No records found to export")) except SPSSIOError as e: return HttpResponseBadRequest(str(e)) else: export = newest_export_for(xform, export_type, options) + export_type = export_type.upper() # log download as well audit_log( - Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Downloaded %(export_type)s export on '%(id_string)s'.") % - {'id_string': xform.id_string, - 'export_type': export_type.upper()}, audit, request) + Actions.EXPORT_DOWNLOADED, + request.user, + owner, + _(f"Downloaded {export_type} export on '{id_string}'."), + audit, + request, + ) if not export.filename and not export.error_message: # tends to happen when using newset_export_for. return HttpResponseNotFound("File does not exist!") - elif not export.filename and export.error_message: + if not export.filename and export.error_message: return HttpResponseBadRequest(str(export.error_message)) # get extension from file_path, exporter could modify to # xlsx if it exceeds limits __, extension = os.path.splitext(export.filename) extension = extension[1:] - if request.GET.get('raw'): + if request.GET.get("raw"): id_string = None response = response_with_mimetype_and_name( Export.EXPORT_MIMES[extension], id_string, extension=extension, - file_path=export.filepath) + file_path=export.filepath, + ) return response -# pylint: disable=R0914 +# pylint: disable=too-many-locals @login_required @require_POST def create_export(request, username, id_string, export_type): @@ -355,15 +391,15 @@ def create_export(request, username, id_string, export_type): Create async export tasks view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) if export_type == Export.EXTERNAL_EXPORT: # check for template before trying to generate a report if not MetaData.external_export(xform): - return HttpResponseForbidden(_(u'No XLS Template set.')) + return HttpResponseForbidden(_("No XLS Template set.")) credential = None if export_type == Export.GOOGLE_SHEETS_EXPORT: @@ -373,80 +409,84 @@ def create_export(request, username, id_string, export_type): return credential query = request.POST.get("query") - force_xlsx = request.POST.get('xls') != 'true' + force_xlsx = request.POST.get("xls") != "true" # export options - group_delimiter = request.POST.get("options[group_delimiter]", '/') - if group_delimiter not in ['.', '/']: - return HttpResponseBadRequest( - _("%s is not a valid delimiter" % group_delimiter)) + group_delimiter = request.POST.get("options[group_delimiter]", "/") + if group_delimiter not in [".", "/"]: + return HttpResponseBadRequest(_(f"{group_delimiter} is not a valid delimiter")) # default is True, so when dont_.. is yes # split_select_multiples becomes False - split_select_multiples = request.POST.get( - "options[dont_split_select_multiples]", "no") == "no" + split_select_multiples = ( + request.POST.get("options[dont_split_select_multiples]", "no") == "no" + ) - binary_select_multiples = getattr(settings, 'BINARY_SELECT_MULTIPLES', - False) + binary_select_multiples = getattr(settings, "BINARY_SELECT_MULTIPLES", False) remove_group_name = request.POST.get("options[remove_group_name]", "false") value_select_multiples = request.POST.get( - "options[value_select_multiples]", "false") + "options[value_select_multiples]", "false" + ) # external export option meta = request.POST.get("meta") options = { - 'group_delimiter': group_delimiter, - 'split_select_multiples': split_select_multiples, - 'binary_select_multiples': binary_select_multiples, - 'value_select_multiples': str_to_bool(value_select_multiples), - 'remove_group_name': str_to_bool(remove_group_name), - 'meta': meta.replace(",", "") if meta else None, - 'google_credentials': credential + "group_delimiter": group_delimiter, + "split_select_multiples": split_select_multiples, + "binary_select_multiples": binary_select_multiples, + "value_select_multiples": str_to_bool(value_select_multiples), + "remove_group_name": str_to_bool(remove_group_name), + "meta": meta.replace(",", "") if meta else None, + "google_credentials": credential, } try: create_async_export(xform, export_type, query, force_xlsx, options) except ExportTypeError: - return HttpResponseBadRequest( - _("%s is not a valid export type" % export_type)) + return HttpResponseBadRequest(_(f"{export_type} is not a valid export type")) else: audit = {"xform": xform.id_string, "export_type": export_type} - audit_log(Actions.EXPORT_CREATED, request.user, owner, - _("Created %(export_type)s export on '%(id_string)s'.") % { - 'export_type': export_type.upper(), - 'id_string': xform.id_string, - }, audit, request) + export_type = export_type.upper() + id_string = xform.id_string + audit_log( + Actions.EXPORT_CREATED, + request.user, + owner, + _(f"Created {export_type} export on '{id_string}'."), + audit, + request, + ) return HttpResponseRedirect( reverse( export_list, kwargs={ "username": username, "id_string": id_string, - "export_type": export_type - })) + "export_type": export_type, + }, + ) + ) def _get_google_credential(request): - token = None - if request.user.is_authenticated: - storage = Storage(TokenStorageModel, 'id', request.user, 'credential') - credential = storage.get() - elif request.session.get('access_token'): - credential = google_client.OAuth2Credentials.from_json(token) - - return credential or HttpResponseRedirect( - google_flow.step1_get_authorize_url()) + google_flow = create_flow() + return HttpResponseRedirect( + google_flow.authorization_url( + access_type="offline", include_granted_scopes="true", prompt="consent" + ) + ) -def export_list(request, username, id_string, export_type): +def export_list(request, username, id_string, export_type): # noqa C901 """ Export list view. """ credential = None - if export_type not in Export.EXPORT_TYPE_DICT: + if export_type.lower() not in Export.EXPORT_TYPE_DICT: return HttpResponseBadRequest( - _(u'Export type "%s" is not supported.' % export_type)) + _(f'Export type "{export_type}" is not supported.') + ) if export_type == Export.GOOGLE_SHEETS_EXPORT: # Retrieve google creds or redirect to google authorization page @@ -454,58 +494,57 @@ def export_list(request, username, id_string, export_type): if isinstance(credential, HttpResponseRedirect): return credential owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) if export_type == Export.EXTERNAL_EXPORT: # check for template before trying to generate a report if not MetaData.external_export(xform): - return HttpResponseForbidden(_(u'No XLS Template set.')) + return HttpResponseForbidden(_("No XLS Template set.")) # Get meta and token - export_token = request.GET.get('token') - export_meta = request.GET.get('meta') + export_token = request.GET.get("token") + export_meta = request.GET.get("meta") options = { - 'group_delimiter': DEFAULT_GROUP_DELIMITER, - 'remove_group_name': False, - 'split_select_multiples': True, - 'binary_select_multiples': False, - 'meta': export_meta, - 'token': export_token, - 'google_credentials': credential + "group_delimiter": DEFAULT_GROUP_DELIMITER, + "remove_group_name": False, + "split_select_multiples": True, + "binary_select_multiples": False, + "meta": export_meta, + "token": export_token, + "google_credentials": credential, } if should_create_new_export(xform, export_type, options): try: create_async_export( - xform, - export_type, - query=None, - force_xlsx=True, - options=options) - except Export.ExportTypeError: + xform, export_type, query=None, force_xlsx=True, options=options + ) + except ExportTypeError: return HttpResponseBadRequest( - _("%s is not a valid export type" % export_type)) + _(f"{export_type} is not a valid export type") + ) - metadata_qs = MetaData.objects.filter(object_id=xform.id, - data_type="external_export")\ - .values('id', 'data_value') + metadata_qs = MetaData.objects.filter( + object_id=xform.id, data_type="external_export" + ).values("id", "data_value") for metadata in metadata_qs: - metadata['data_value'] = metadata.get('data_value').split('|')[0] + metadata["data_value"] = metadata.get("data_value").split("|")[0] data = { - 'username': owner.username, - 'xform': xform, - 'export_type': export_type, - 'export_type_name': Export.EXPORT_TYPE_DICT[export_type], - 'exports': Export.objects.filter( - xform=xform, export_type=export_type).order_by('-created_on'), - 'metas': metadata_qs + "username": owner.username, + "xform": xform, + "export_type": export_type, + "export_type_name": Export.EXPORT_TYPE_DICT[export_type], + "exports": Export.objects.filter(xform=xform, export_type=export_type).order_by( + "-created_on" + ), + "metas": metadata_qs, } # yapf: disable - return render(request, 'export_list.html', data) + return render(request, "export_list.html", data) def export_progress(request, username, id_string, export_type): @@ -513,47 +552,52 @@ def export_progress(request, username, id_string, export_type): Async export progress view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) # find the export entry in the db - export_ids = request.GET.getlist('export_ids') + export_ids = request.GET.getlist("export_ids") exports = Export.objects.filter( - xform=xform, id__in=export_ids, export_type=export_type) + xform=xform, id__in=export_ids, export_type=export_type + ) statuses = [] for export in exports: status = { - 'complete': False, - 'url': None, - 'filename': None, - 'export_id': export.id + "complete": False, + "url": None, + "filename": None, + "export_id": export.id, } if export.status == Export.SUCCESSFUL: - status['url'] = reverse( + status["url"] = reverse( export_download, kwargs={ - 'username': owner.username, - 'id_string': xform.id_string, - 'export_type': export.export_type, - 'filename': export.filename - }) - status['filename'] = export.filename - if export.export_type == Export.GOOGLE_SHEETS_EXPORT and \ - export.export_url is None: - status['url'] = None - if export.export_type == Export.EXTERNAL_EXPORT \ - and export.export_url is None: - status['url'] = None + "username": owner.username, + "id_string": xform.id_string, + "export_type": export.export_type, + "filename": export.filename, + }, + ) + status["filename"] = export.filename + if ( + export.export_type == Export.GOOGLE_SHEETS_EXPORT + and export.export_url is None + ): + status["url"] = None + if ( + export.export_type == Export.EXTERNAL_EXPORT + and export.export_url is None + ): + status["url"] = None # mark as complete if it either failed or succeeded but NOT pending - if export.status == Export.SUCCESSFUL \ - or export.status == Export.FAILED: - status['complete'] = True + if export.status in [Export.SUCCESSFUL, Export.FAILED]: + status["complete"] = True statuses.append(status) - return HttpResponse(json.dumps(statuses), content_type='application/json') + return JsonResponse(statuses, safe=False) def export_download(request, username, id_string, export_type, filename): @@ -561,31 +605,37 @@ def export_download(request, username, id_string, export_type, filename): Export download view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) helper_auth_helper(request) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) # find the export entry in the db export = get_object_or_404(Export, xform=xform, filename=filename) - if (export_type == Export.GOOGLE_SHEETS_EXPORT or - export_type == Export.EXTERNAL_EXPORT) and \ - export.export_url is not None: + is_external_export = export_type in [ + Export.GOOGLE_SHEETS_EXPORT, + Export.EXTERNAL_EXPORT, + ] + if is_external_export and export.export_url is not None: return HttpResponseRedirect(export.export_url) ext, mime_type = export_def_from_filename(export.filename) + export_type = export.export_type.upper() + filename = export.filename + id_string = xform.id_string audit = {"xform": xform.id_string, "export_type": export.export_type} - audit_log(Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Downloaded %(export_type)s export '%(filename)s' " - "on '%(id_string)s'.") % { - 'export_type': export.export_type.upper(), - 'filename': export.filename, - 'id_string': xform.id_string, - }, audit, request) - if request.GET.get('raw'): + audit_log( + Actions.EXPORT_DOWNLOADED, + request.user, + owner, + _(f"Downloaded {export_type} export '{filename}' on '{id_string}'."), + audit, + request, + ) + if request.GET.get("raw"): id_string = None default_storage = get_storage_class()() @@ -597,7 +647,8 @@ def export_download(request, username, id_string, export_type, filename): name=basename, extension=ext, file_path=export.filepath, - show_date=False) + show_date=False, + ) return response @@ -608,33 +659,39 @@ def delete_export(request, username, id_string, export_type): Delete export view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) - export_id = request.POST.get('export_id') + export_id = request.POST.get("export_id") # find the export entry in the db export = get_object_or_404(Export, id=export_id) - + export_type = export.export_type.upper() export.delete() + + filename = export.filename + id_string = xform.id_string audit = {"xform": xform.id_string, "export_type": export.export_type} - audit_log(Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Deleted %(export_type)s export '%(filename)s'" - " on '%(id_string)s'.") % { - 'export_type': export.export_type.upper(), - 'filename': export.filename, - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.EXPORT_DOWNLOADED, + request.user, + owner, + _(f"Deleted {export_type} export '{filename}' on '{id_string}'."), + audit, + request, + ) return HttpResponseRedirect( reverse( export_list, kwargs={ "username": username, "id_string": id_string, - "export_type": export_type - })) + "export_type": export_type, + }, + ) + ) def zip_export(request, username, id_string): @@ -642,39 +699,44 @@ def zip_export(request, username, id_string): Zip export view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) helper_auth_helper(request) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) - if request.GET.get('raw'): + return HttpResponseForbidden(_("Not shared.")) + if request.GET.get("raw"): id_string = None attachments = Attachment.objects.filter(instance__xform=xform) zip_file = None - try: - zip_file = create_attachments_zipfile(attachments) + with NamedTemporaryFile() as zip_file: + create_attachments_zipfile(attachments, zip_file) audit = {"xform": xform.id_string, "export_type": Export.ZIP_EXPORT} - audit_log(Actions.EXPORT_CREATED, request.user, owner, - _("Created ZIP export on '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.EXPORT_CREATED, + request.user, + owner, + _(f"Created ZIP export on '{xform.id_string}'."), + audit, + request, + ) # log download as well - audit_log(Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Downloaded ZIP export on '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) - if request.GET.get('raw'): + audit_log( + Actions.EXPORT_DOWNLOADED, + request.user, + owner, + _(f"Downloaded ZIP export on '{xform.id_string}'."), + audit, + request, + ) + if request.GET.get("raw"): id_string = None - response = response_with_mimetype_and_name('zip', id_string) + response = response_with_mimetype_and_name("zip", id_string) response.write(FileWrapper(zip_file)) - response['Content-Length'] = zip_file.tell() + response["Content-Length"] = zip_file.tell() zip_file.seek(0) - finally: - if zip_file: - zip_file.close() return response @@ -685,29 +747,36 @@ def kml_export(request, username, id_string): """ # read the locations from the database owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) helper_auth_helper(request) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) - data = {'data': kml_export_data(id_string, user=owner, xform=xform)} + return HttpResponseForbidden(_("Not shared.")) + data = {"data": kml_export_data(id_string, user=owner, xform=xform)} response = render( - request, - "survey.kml", - data, - content_type="application/vnd.google-earth.kml+xml") - response['Content-Disposition'] = \ - generate_content_disposition_header(id_string, 'kml') + request, "survey.kml", data, content_type="application/vnd.google-earth.kml+xml" + ) + response["Content-Disposition"] = generate_content_disposition_header( + id_string, "kml" + ) audit = {"xform": xform.id_string, "export_type": Export.KML_EXPORT} - audit_log(Actions.EXPORT_CREATED, request.user, owner, - _("Created KML export on '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.EXPORT_CREATED, + request.user, + owner, + _(f"Created KML export on '{xform.id_string}'."), + audit, + request, + ) # log download as well - audit_log(Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Downloaded KML export on '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.EXPORT_DOWNLOADED, + request.user, + owner, + _(f"Downloaded KML export on '{xform.id_string}'."), + audit, + request, + ) return response @@ -725,40 +794,47 @@ def google_xls_export(request, username, id_string): pass else: token = token_storage.token - elif request.session.get('access_token'): - token = request.session.get('access_token') + elif request.session.get("access_token"): + token = request.session.get("access_token") if token is None: request.session["google_redirect_url"] = reverse( - google_xls_export, - kwargs={'username': username, - 'id_string': id_string}) - return HttpResponseRedirect(google_flow.step1_get_authorize_url()) + google_xls_export, kwargs={"username": username, "id_string": id_string} + ) + google_flow = create_flow() + return HttpResponseRedirect( + google_flow.authorization_url( + access_type="offline", include_granted_scopes="true", prompt="consent" + ) + ) owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) - is_valid, data_dictionary = set_instances_for_export( - id_string, owner, request) + is_valid, data_dictionary = set_instances_for_export(id_string, owner, request) if not is_valid: return data_dictionary xls_writer = XlsWriter() - tmp = NamedTemporaryFile(delete=False) - xls_writer.set_file(tmp) - xls_writer.set_data_dictionary(data_dictionary) - temp_file = xls_writer.save_workbook_to_file() - temp_file.close() + with NamedTemporaryFile(delete=False) as tmp: + xls_writer.set_file(tmp) + xls_writer.set_data_dictionary(data_dictionary) + temp_file = xls_writer.save_workbook_to_file() + temp_file.close() url = None os.unlink(tmp.name) audit = {"xform": xform.id_string, "export_type": "google"} - audit_log(Actions.EXPORT_CREATED, request.user, owner, - _("Created Google Docs export on '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.EXPORT_CREATED, + request.user, + owner, + _(f"Created Google Docs export on '{xform.id_string}'."), + audit, + request, + ) return HttpResponseRedirect(url) @@ -768,101 +844,120 @@ def data_view(request, username, id_string): Data view displays submission data. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'id_string__iexact': id_string, 'user': owner}) + xform = get_form({"id_string__iexact": id_string, "user": owner}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) - data = {'owner': owner, 'xform': xform} + data = {"owner": owner, "xform": xform} audit = { "xform": xform.id_string, } - audit_log(Actions.FORM_DATA_VIEWED, request.user, owner, - _("Requested data view for '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.FORM_DATA_VIEWED, + request.user, + owner, + _(f"Requested data view for '{xform.id_string}'."), + audit, + request, + ) return render(request, "data_view.html", data) -def attachment_url(request, size='medium'): +def attachment_url(request, size="medium"): """ Redirects to image attachment of the specified size, defaults to 'medium'. """ - media_file = request.GET.get('media_file') - no_redirect = request.GET.get('no_redirect') + media_file = request.GET.get("media_file") + no_redirect = request.GET.get("no_redirect") if not media_file: - return HttpResponseNotFound(_(u'Attachment not found')) + return HttpResponseNotFound(_("Attachment not found")) result = Attachment.objects.filter(media_file=media_file).order_by()[0:1] if not result: - return HttpResponseNotFound(_(u'Attachment not found')) + return HttpResponseNotFound(_("Attachment not found")) attachment = result[0] - if size == 'original' and no_redirect == 'true': + if size == "original" and no_redirect == "true": response = response_with_mimetype_and_name( attachment.mimetype, attachment.name, extension=attachment.extension, - file_path=attachment.media_file.name) + file_path=attachment.media_file.name, + ) return response - if not attachment.mimetype.startswith('image'): + if not attachment.mimetype.startswith("image"): return redirect(attachment.media_file.url) media_url = image_url(attachment, size) if media_url: return redirect(media_url) - return HttpResponseNotFound(_(u'Error: Attachment not found')) + return HttpResponseNotFound(_("Error: Attachment not found")) def instance(request, username, id_string): """ Data view for browsing submissions one at a time. """ - # pylint: disable=W0612 - xform, is_owner, can_edit, can_view = get_xform_and_perms( - username, id_string, request) + xform, _is_owner, can_edit, can_view = get_xform_and_perms( + username, id_string, request + ) # no access - if not (xform.shared_data or can_view - or request.session.get('public_link') == xform.uuid): - return HttpResponseForbidden(_(u'Not shared.')) + if not ( + xform.shared_data + or can_view + or request.session.get("public_link") == xform.uuid + ): + return HttpResponseForbidden(_("Not shared.")) audit = { "xform": xform.id_string, } - audit_log(Actions.FORM_DATA_VIEWED, request.user, xform.user, - _("Requested instance view for '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.FORM_DATA_VIEWED, + request.user, + xform.user, + _(f"Requested instance view for '{xform.id_string}'."), + audit, + request, + ) - return render(request, 'instance.html', { - 'username': username, - 'id_string': id_string, - 'xform': xform, - 'can_edit': can_edit - }) + return render( + request, + "instance.html", + { + "username": username, + "id_string": id_string, + "xform": xform, + "can_edit": can_edit, + }, + ) def charts(request, username, id_string): """ Charts view. """ - # pylint: disable=W0612 - xform, is_owner, can_edit, can_view = get_xform_and_perms( - username, id_string, request) + xform, _is_owner, _can_edit, can_view = get_xform_and_perms( + username, id_string, request + ) # no access - if not (xform.shared_data or can_view - or request.session.get('public_link') == xform.uuid): - return HttpResponseForbidden(_(u'Not shared.')) + if not ( + xform.shared_data + or can_view + or request.session.get("public_link") == xform.uuid + ): + return HttpResponseForbidden(_("Not shared.")) try: - lang_index = int(request.GET.get('lang', 0)) + lang_index = int(request.GET.get("lang", 0)) except ValueError: lang_index = 0 try: - page = int(request.GET.get('page', 0)) + page = int(request.GET.get("page", 0)) except ValueError: page = 0 else: @@ -871,26 +966,28 @@ def charts(request, username, id_string): summaries = build_chart_data(xform, lang_index, page) if request.is_ajax(): - template = 'charts_snippet.html' + template = "charts_snippet.html" else: - template = 'charts.html' + template = "charts.html" - return render(request, template, - {'xform': xform, - 'summaries': summaries, - 'page': page + 1}) + return render( + request, template, {"xform": xform, "summaries": summaries, "page": page + 1} + ) def stats_tables(request, username, id_string): """ Stats view. """ - # pylint: disable=W0612 - xform, is_owner, can_edit, can_view = get_xform_and_perms( - username, id_string, request) + xform, _is_owner, _can_edit, can_view = get_xform_and_perms( + username, id_string, request + ) # no access - if not (xform.shared_data or can_view - or request.session.get('public_link') == xform.uuid): - return HttpResponseForbidden(_(u'Not shared.')) - - return render(request, 'stats_tables.html', {'xform': xform}) + if not ( + xform.shared_data + or can_view + or request.session.get("public_link") == xform.uuid + ): + return HttpResponseForbidden(_("Not shared.")) + + return render(request, "stats_tables.html", {"xform": xform}) diff --git a/onadata/libs/authentication.py b/onadata/libs/authentication.py index d79bc549c2..739a6cffae 100644 --- a/onadata/libs/authentication.py +++ b/onadata/libs/authentication.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- -"""Authentication classes. +""" +Authentication classes. """ from __future__ import unicode_literals @@ -7,16 +8,19 @@ from typing import Optional, Tuple from django.conf import settings -from django.contrib.auth.models import User, update_last_login +from django.contrib.auth import get_user_model +from django.contrib.auth.models import update_last_login from django.core.signing import BadSignature from django.db import DataError from django.utils import timezone -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ import jwt from django_digest import HttpDigestAuthenticator from multidb.pinning import use_master from oauth2_provider.models import AccessToken +from oauth2_provider.oauth2_validators import OAuth2Validator +from oauth2_provider.settings import oauth2_settings from rest_framework import exceptions from rest_framework.authentication import ( BaseAuthentication, @@ -25,8 +29,6 @@ ) from rest_framework.authtoken.models import Token from rest_framework.exceptions import AuthenticationFailed -from oauth2_provider.oauth2_validators import OAuth2Validator -from oauth2_provider.settings import oauth2_settings from oidc.utils import authenticate_sso from onadata.apps.api.models.temp_token import TempToken @@ -61,6 +63,9 @@ ], ) +# pylint: disable=invalid-name +User = get_user_model() + def expired(time_token_created): """Checks if the time between when time_token_created and current time @@ -72,7 +77,7 @@ def expired(time_token_created): time_diff = (timezone.now() - time_token_created).total_seconds() token_expiry_time = TEMP_TOKEN_EXPIRY_TIME - return True if time_diff > token_expiry_time else False + return time_diff > token_expiry_time def get_api_token(cookie_jwt): @@ -86,11 +91,11 @@ def get_api_token(cookie_jwt): api_token = Token.objects.get(key=jwt_payload.get(API_TOKEN)) return api_token except BadSignature as e: - raise exceptions.AuthenticationFailed(_("Bad Signature: %s" % e)) + raise exceptions.AuthenticationFailed(_(f"Bad Signature: {e}")) from e except jwt.DecodeError as e: - raise exceptions.AuthenticationFailed(_("JWT DecodeError: %s" % e)) - except Token.DoesNotExist: - raise exceptions.AuthenticationFailed(_("Invalid token")) + raise exceptions.AuthenticationFailed(_(f"JWT DecodeError: {e}")) from e + except Token.DoesNotExist as e: + raise exceptions.AuthenticationFailed(_("Invalid token")) from e class DigestAuthentication(BaseAuthentication): @@ -110,24 +115,20 @@ def authenticate(self, request): if self.authenticator.authenticate(request): update_last_login(None, request.user) return request.user, None - else: - attempts = login_attempts(request) - remaining_attempts = ( - getattr(settings, "MAX_LOGIN_ATTEMPTS", 10) - attempts - ) - raise AuthenticationFailed( - _( - "Invalid username/password. " - "For security reasons, after {} more failed " - "login attempts you'll have to wait {} minutes " - "before trying again.".format( - remaining_attempts, - getattr(settings, "LOCKOUT_TIME", 1800) // 60, - ) - ) + attempts = login_attempts(request) + remaining_attempts = getattr(settings, "MAX_LOGIN_ATTEMPTS", 10) - attempts + # pylint: disable=unused-variable + lockout_time = getattr(settings, "LOCKOUT_TIME", 1800) // 60 # noqa + raise AuthenticationFailed( + _( + "Invalid username/password. " + f"For security reasons, after {remaining_attempts} more failed " + f"login attempts you'll have to wait {lockout_time} minutes " + "before trying again." ) + ) except (AttributeError, ValueError, DataError) as e: - raise AuthenticationFailed(e) + raise AuthenticationFailed(e) from e def authenticate_header(self, request): response = self.authenticator.build_challenge_response() @@ -149,7 +150,7 @@ def authenticate(self, request): if len(auth) == 1: error_msg = _("Invalid token header. No credentials provided.") raise exceptions.AuthenticationFailed(error_msg) - elif len(auth) > 2: + if len(auth) > 2: error_msg = _( "Invalid token header. " "Token string should not contain spaces." ) @@ -162,7 +163,7 @@ def authenticate_credentials(self, key): if isinstance(key, bytes): key = key.decode("utf-8") token = self.model.objects.get(key=key) - except self.model.DoesNotExist: + except self.model.DoesNotExist as e: invalid_token = True if getattr(settings, "SLAVE_DATABASES", []): try: @@ -173,7 +174,7 @@ def authenticate_credentials(self, key): else: invalid_token = False if invalid_token: - raise exceptions.AuthenticationFailed(_("Invalid token")) + raise exceptions.AuthenticationFailed(_("Invalid token")) from e if not token.user.is_active: raise exceptions.AuthenticationFailed(_("User inactive or deleted")) @@ -201,11 +202,11 @@ def authenticate(self, request): if getattr(api_token, "user"): return api_token.user, api_token - except self.model.DoesNotExist: - raise exceptions.AuthenticationFailed(_("Invalid token")) + except self.model.DoesNotExist as e: + raise exceptions.AuthenticationFailed(_("Invalid token")) from e except KeyError: pass - except BadSignature: + except BadSignature as e: # if the cookie wasn't signed it means zebra might have # generated it cookie_jwt = request.COOKIES.get(ENKETO_AUTH_COOKIE) @@ -215,7 +216,7 @@ def authenticate(self, request): raise exceptions.ParseError( _("Malformed cookie. Clear your cookies then try again") - ) + ) from e return None @@ -244,26 +245,29 @@ def authenticate(self, request): # pylint: disable=no-self-use def retrieve_user_identification(request) -> Tuple[Optional[str], Optional[str]]: - ip = None + """ + Retrieve user information from a HTTP request. + """ + ip_address = None - if request.META.get("HTTP_X_REAL_IP"): - ip = request.META["HTTP_X_REAL_IP"].split(",")[0] + if request.headers.get("X-Real-Ip"): + ip_address = request.headers["X-Real-Ip"].split(",")[0] else: - ip = request.META.get("REMOTE_ADDR") + ip_address = request.META.get("REMOTE_ADDR") try: - if isinstance(request.META["HTTP_AUTHORIZATION"], bytes): + if isinstance(request.headers["Authorization"], bytes): username = ( - request.META["HTTP_AUTHORIZATION"].decode("utf-8").split('"')[1].strip() + request.headers["Authorization"].decode("utf-8").split('"')[1].strip() ) else: - username = request.META["HTTP_AUTHORIZATION"].split('"')[1].strip() + username = request.headers["Authorization"].split('"')[1].strip() except (TypeError, AttributeError, IndexError): pass else: if not username: raise AuthenticationFailed(_("Invalid username")) - return ip, username + return ip_address, username return None, None @@ -276,10 +280,10 @@ def check_lockout(request) -> Tuple[Optional[str], Optional[str]]: """ uri_path = request.get_full_path() if not any(part in LOCKOUT_EXCLUDED_PATHS for part in uri_path.split("/")): - ip, username = retrieve_user_identification(request) + ip_address, username = retrieve_user_identification(request) - if ip and username: - lockout = cache.get(safe_key("{}{}-{}".format(LOCKOUT_IP, ip, username))) + if ip_address and username: + lockout = cache.get(safe_key(f"{LOCKOUT_IP}{ip_address}-{username}")) if lockout: time_locked_out = datetime.now() - datetime.strptime( lockout, "%Y-%m-%dT%H:%M:%S" @@ -290,12 +294,11 @@ def check_lockout(request) -> Tuple[Optional[str], Optional[str]]: ) raise AuthenticationFailed( _( - "Locked out. Too many wrong username" - "/password attempts. " - "Try again in {} minutes.".format(remaining_time) + "Locked out. Too many wrong username/password attempts. " + f"Try again in {remaining_time} minutes." ) ) - return ip, username + return ip_address, username return None, None @@ -304,18 +307,18 @@ def login_attempts(request): Track number of login attempts made by a specific IP within a specified amount of time """ - ip, username = check_lockout(request) - attempts_key = safe_key("{}{}-{}".format(LOGIN_ATTEMPTS, ip, username)) + ip_address, username = check_lockout(request) + attempts_key = safe_key(f"{LOGIN_ATTEMPTS}{ip_address}-{username}") attempts = cache.get(attempts_key) if attempts: cache.incr(attempts_key) attempts = cache.get(attempts_key) if attempts >= getattr(settings, "MAX_LOGIN_ATTEMPTS", 10): - lockout_key = safe_key("{}{}-{}".format(LOCKOUT_IP, ip, username)) + lockout_key = safe_key(f"{LOCKOUT_IP}{ip_address}-{username}") lockout = cache.get(lockout_key) if not lockout: - send_lockout_email(username, ip) + send_lockout_email(username, ip_address) cache.set( lockout_key, datetime.now().strftime("%Y-%m-%dT%H:%M:%S"), @@ -330,14 +333,17 @@ def login_attempts(request): return cache.get(attempts_key) -def send_lockout_email(username, ip): +def send_lockout_email(username, ip_address): + """ + Send locked out email + """ try: user = User.objects.get(username=username) except User.DoesNotExist: pass else: - email_data = get_account_lockout_email_data(username, ip) - end_email_data = get_account_lockout_email_data(username, ip, end=True) + email_data = get_account_lockout_email_data(username, ip_address) + end_email_data = get_account_lockout_email_data(username, ip_address, end=True) send_account_lockout_email.apply_async( args=[ @@ -404,6 +410,7 @@ def validate_bearer_token(self, token, scopes, request): request.scopes = scopes request.access_token = access_token return True - else: - self._set_oauth2_error_on_request(request, access_token, scopes) - return False + + self._set_oauth2_error_on_request(request, access_token, scopes) + + return False diff --git a/onadata/libs/data/__init__.py b/onadata/libs/data/__init__.py index aec01f3cd4..a8efc0e3a5 100644 --- a/onadata/libs/data/__init__.py +++ b/onadata/libs/data/__init__.py @@ -16,3 +16,20 @@ def parse_int(num): return num and int(num) except ValueError: pass + return None + + +# source from deprecated module distutils/util.py +def strtobool(val): + """Convert a string representation of truth to true (1) or false (0). + + True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values + are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if + 'val' is anything else. + """ + val = val.lower() + if val in ("y", "yes", "t", "true", "on", "1"): + return 1 + if val in ("n", "no", "f", "false", "off", "0"): + return 0 + raise ValueError(f"invalid truth value {val}") diff --git a/onadata/libs/data/query.py b/onadata/libs/data/query.py index 372238f04a..c58a19a4e6 100644 --- a/onadata/libs/data/query.py +++ b/onadata/libs/data/query.py @@ -263,4 +263,4 @@ def is_date_field(xform, field): @property def using_postgres(): return settings.DATABASES[ - 'default']['ENGINE'] == 'django.db.backends.postgresql_psycopg2' + 'default']['ENGINE'] == 'django.db.backends.postgresql' diff --git a/onadata/libs/exceptions.py b/onadata/libs/exceptions.py index da0876fe25..d41c22f8de 100644 --- a/onadata/libs/exceptions.py +++ b/onadata/libs/exceptions.py @@ -1,35 +1,37 @@ -from django.utils.translation import ugettext_lazy as _ +# -*- coding: utf-8 -*- +"""Custom Expecting classes.""" +from django.utils.translation import gettext_lazy as _ from rest_framework.exceptions import APIException class EnketoError(Exception): + """Enketo specigic exceptions""" - default_message = _("There was a problem with your submissionor" - " form. Please contact support.") + default_message = _( + "There was a problem with your submission or form. Please contact support." + ) def __init__(self, message=None): - if message is None: - self.message = self.default_message - else: - self.message = message - - def __str__(self): - return "{}".format(self.message) + self.message = message if message is not None else self.default_message + super().__init__(self.message) class NoRecordsFoundError(Exception): - pass + """Raise for when no records are found.""" class NoRecordsPermission(Exception): - pass + """Raise when no permissions to access records.""" class J2XException(Exception): - pass + """Raise for json-to-xls exceptions on external exports.""" +# pylint: disable=too-few-public-methods class ServiceUnavailable(APIException): + """Custom service unavailable exception.""" + status_code = 503 - default_detail = 'Service temporarily unavailable, try again later.' + default_detail = "Service temporarily unavailable, try again later." diff --git a/onadata/libs/filters.py b/onadata/libs/filters.py index 9c14884361..78ad7d7c1f 100644 --- a/onadata/libs/filters.py +++ b/onadata/libs/filters.py @@ -1,12 +1,17 @@ +# -*- coding: utf-8 -*- +""" +Django rest_framework ViewSet filters. +""" from uuid import UUID -from django.contrib.auth.models import User +import six + +from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ObjectDoesNotExist from django.db.models import Q from django.http import Http404 from django.shortcuts import get_object_or_404 -from django.utils import six from django_filters import rest_framework as django_filter_filters from rest_framework import filters from rest_framework_guardian.filters import ObjectPermissionsFilter @@ -16,18 +21,31 @@ from onadata.apps.viewer.models import Export from onadata.libs.utils.numeric import int_or_parse_error from onadata.libs.utils.common_tags import MEDIA_FILE_TYPES -from onadata.libs.permissions import \ - exclude_items_from_queryset_using_xform_meta_perms +from onadata.libs.permissions import exclude_items_from_queryset_using_xform_meta_perms + + +# pylint: disable=invalid-name +User = get_user_model() + + +def _is_public_xform(export_id: int): + export = Export.objects.filter(pk=export_id).first() + + if export: + return export.xform.shared_data or export.xform.shared + + return False +# pylint: disable=too-few-public-methods class AnonDjangoObjectPermissionFilter(ObjectPermissionsFilter): + """Anonymous user permission filter class.""" def filter_queryset(self, request, queryset, view): """ Anonymous user has no object permissions, return queryset as it is. """ - form_id = view.kwargs.get( - view.lookup_field, view.kwargs.get('xform_pk')) + form_id = view.kwargs.get(view.lookup_field, view.kwargs.get("xform_pk")) lookup_field = view.lookup_field queryset = queryset.filter(deleted_at=None) @@ -35,32 +53,28 @@ def filter_queryset(self, request, queryset, view): return queryset if form_id: - if lookup_field == 'pk': - int_or_parse_error(form_id, - u'Invalid form ID. It must be a positive' - ' integer') + if lookup_field == "pk": + int_or_parse_error( + form_id, "Invalid form ID. It must be a positive" " integer" + ) try: - if lookup_field == 'uuid': + if lookup_field == "uuid": form_id = UUID(form_id) - form = queryset.get( - Q(uuid=form_id.hex) | Q(uuid=str(form_id))) + form = queryset.get(Q(uuid=form_id.hex) | Q(uuid=str(form_id))) else: xform_kwargs = {lookup_field: form_id} form = queryset.get(**xform_kwargs) - except ObjectDoesNotExist: - raise Http404 + except ObjectDoesNotExist as non_existent_object: + raise Http404 from non_existent_object # Check if form is public and return it if form.shared: - if lookup_field == 'uuid': - return queryset.filter( - Q(uuid=form_id.hex) | Q(uuid=str(form_id))) - else: - return queryset.filter(Q(**xform_kwargs)) + if lookup_field == "uuid": + return queryset.filter(Q(uuid=form_id.hex) | Q(uuid=str(form_id))) + return queryset.filter(Q(**xform_kwargs)) - return super(AnonDjangoObjectPermissionFilter, self)\ - .filter_queryset(request, queryset, view) + return super().filter_queryset(request, queryset, view) # pylint: disable=too-few-public-methods @@ -73,20 +87,27 @@ class EnketoAnonDjangoObjectPermissionFilter(AnonDjangoObjectPermissionFilter): def filter_queryset(self, request, queryset, view): """Check report_xform permission when requesting for Enketo URL.""" - if view.action == 'enketo': - self.perm_format = '%(app_label)s.report_%(model_name)s' # noqa pylint: disable=W0201 - return super(EnketoAnonDjangoObjectPermissionFilter, self)\ - .filter_queryset(request, queryset, view) + if view.action == "enketo": + # noqa pylint: disable=attribute-defined-outside-init + self.perm_format = "%(app_label)s.report_%(model_name)s" + + return super().filter_queryset(request, queryset, view) +# pylint: disable=too-few-public-methods class XFormListObjectPermissionFilter(AnonDjangoObjectPermissionFilter): - perm_format = '%(app_label)s.report_%(model_name)s' + """XFormList permission filter with using [app].report_[model] form.""" + + perm_format = "%(app_label)s.report_%(model_name)s" -class XFormListXFormPKFilter(object): +class XFormListXFormPKFilter: + """Filter forms via 'xform_pk' param.""" + # pylint: disable=no-self-use def filter_queryset(self, request, queryset, view): - xform_pk = view.kwargs.get('xform_pk') + """Returns an XForm queryset filtered by the 1xform_pk' param.""" + xform_pk = view.kwargs.get("xform_pk") if xform_pk: try: xform_pk = int(xform_pk) @@ -101,51 +122,65 @@ def filter_queryset(self, request, queryset, view): class FormIDFilter(django_filter_filters.FilterSet): - formID = django_filter_filters.CharFilter(field_name="id_string") + """formID filter using the XForm.id_string.""" + + # pylint: disable=invalid-name + formID = django_filter_filters.CharFilter(field_name="id_string") # noqa + # pylint: disable=missing-class-docstring class Meta: model = XForm - fields = ['formID'] + fields = ["formID"] +# pylint: disable=too-few-public-methods class OrganizationPermissionFilter(ObjectPermissionsFilter): + """Organization profiles filter + + Based on the organization the profile is added to. + """ def filter_queryset(self, request, queryset, view): """Return a filtered queryset or all profiles if a getting a specific - profile.""" - if view.action == 'retrieve' and request.method == 'GET': + profile.""" + if view.action == "retrieve" and request.method == "GET": return queryset.model.objects.all() - filtered_queryset = super(OrganizationPermissionFilter, self)\ - .filter_queryset(request, queryset, view) - org_users = set([group.team.organization - for group in request.user.groups.all()] + [ - o.user for o in filtered_queryset]) + filtered_queryset = super().filter_queryset(request, queryset, view) + org_users = set( + [group.team.organization for group in request.user.groups.all()] + + [o.user for o in filtered_queryset] + ) - return queryset.model.objects.filter(user__in=org_users, - user__is_active=True) + return queryset.model.objects.filter(user__in=org_users, user__is_active=True) +# pylint: disable=too-few-public-methods class XFormOwnerFilter(filters.BaseFilterBackend): + """XForm `owner` filter""" - owner_prefix = 'user' + owner_prefix = "user" + # pylint: disable=unused-argument def filter_queryset(self, request, queryset, view): - owner = request.query_params.get('owner') + """Filter by `owner` query parameter.""" + owner = request.query_params.get("owner") if owner: - kwargs = { - self.owner_prefix + '__username__iexact': owner - } + kwargs = {self.owner_prefix + "__username__iexact": owner} return queryset.filter(**kwargs) return queryset +# pylint: disable=too-few-public-methods class DataFilter(ObjectPermissionsFilter): + """Shared data filter.""" + # pylint: disable=unused-argument def filter_queryset(self, request, queryset, view): + """Filter by ``XForm.shared_data = True`` for anonymous users.""" if request.user.is_anonymous: return queryset.filter(Q(shared_data=True)) return queryset @@ -155,57 +190,85 @@ class InstanceFilter(django_filter_filters.FilterSet): """ Instance FilterSet implemented using django-filter """ + submitted_by__id = django_filter_filters.ModelChoiceFilter( - field_name='user', + field_name="user", queryset=User.objects.all(), - to_field_name='id', + to_field_name="id", ) submitted_by__username = django_filter_filters.ModelChoiceFilter( - field_name='user', + field_name="user", queryset=User.objects.all(), - to_field_name='username', + to_field_name="username", ) media_all_received = django_filter_filters.BooleanFilter() + # pylint: disable=missing-class-docstring class Meta: model = Instance - date_field_lookups = ['exact', 'gt', 'lt', 'gte', 'lte', 'year', - 'year__gt', 'year__lt', 'year__gte', 'year__lte', - 'month', 'month__gt', 'month__lt', 'month__gte', - 'month__lte', 'day', 'day__gt', 'day__lt', - 'day__gte', 'day__lte'] - generic_field_lookups = ['exact', 'gt', 'lt', 'gte', 'lte'] - fields = {'date_created': date_field_lookups, - 'date_modified': date_field_lookups, - 'last_edited': date_field_lookups, - 'media_all_received': ['exact'], - 'status': ['exact'], - 'survey_type__slug': ['exact'], - 'user__id': ['exact'], - 'user__username': ['exact'], - 'uuid': ['exact'], - 'version': generic_field_lookups} + date_field_lookups = [ + "exact", + "gt", + "lt", + "gte", + "lte", + "year", + "year__gt", + "year__lt", + "year__gte", + "year__lte", + "month", + "month__gt", + "month__lt", + "month__gte", + "month__lte", + "day", + "day__gt", + "day__lt", + "day__gte", + "day__lte", + ] + generic_field_lookups = ["exact", "gt", "lt", "gte", "lte"] + fields = { + "date_created": date_field_lookups, + "date_modified": date_field_lookups, + "last_edited": date_field_lookups, + "media_all_received": ["exact"], + "status": ["exact"], + "survey_type__slug": ["exact"], + "user__id": ["exact"], + "user__username": ["exact"], + "uuid": ["exact"], + "version": generic_field_lookups, + } +# pylint: disable=too-few-public-methods class ProjectOwnerFilter(filters.BaseFilterBackend): - owner_prefix = 'organization' + """Project `owner` filter.""" + + owner_prefix = "organization" + # pylint: disable=unused-argument def filter_queryset(self, request, queryset, view): - owner = request.query_params.get('owner') + """Project `owner` filter.""" + owner = request.query_params.get("owner") if owner: - kwargs = { - self.owner_prefix + '__username__iexact': owner - } + kwargs = {self.owner_prefix + "__username__iexact": owner} return queryset.filter(**kwargs) | Project.objects.filter( - shared=True, deleted_at__isnull=True, **kwargs) + shared=True, deleted_at__isnull=True, **kwargs + ) return queryset +# pylint: disable=too-few-public-methods class AnonUserProjectFilter(ObjectPermissionsFilter): - owner_prefix = 'organization' + """Anonymous user project filter.""" + + owner_prefix = "organization" def filter_queryset(self, request, queryset, view): """ @@ -218,51 +281,56 @@ def filter_queryset(self, request, queryset, view): return queryset.filter(Q(shared=True)) if project_id: - int_or_parse_error(project_id, - u"Invalid value for project_id. It must be a" - " positive integer.") + int_or_parse_error( + project_id, + "Invalid value for project_id. It must be a positive integer.", + ) # check if project is public and return it try: project = queryset.get(id=project_id) - except ObjectDoesNotExist: - raise Http404 + except ObjectDoesNotExist as non_existent_object: + raise Http404 from non_existent_object if project.shared: return queryset.filter(Q(id=project_id)) - return super(AnonUserProjectFilter, self)\ - .filter_queryset(request, queryset, view) + return super().filter_queryset(request, queryset, view) +# pylint: disable=too-few-public-methods class TagFilter(filters.BaseFilterBackend): + """Tag filter using the `tags` query parameter.""" + # pylint: disable=unused-argument def filter_queryset(self, request, queryset, view): + """Tag filter using the `tags` query parameter.""" # filter by tags if available. - tags = request.query_params.get('tags', None) + tags = request.query_params.get("tags", None) if tags and isinstance(tags, six.string_types): - tags = tags.split(',') + tags = tags.split(",") return queryset.filter(tags__name__in=tags) return queryset -class XFormPermissionFilterMixin(object): +# pylint: disable=too-few-public-methods +class XFormPermissionFilterMixin: + """XForm permission filter.""" def _xform_filter(self, request, view, keyword): """Use XForm permissions""" - xform = request.query_params.get('xform') + xform = request.query_params.get("xform") public_forms = XForm.objects.none() if xform: - int_or_parse_error(xform, - u"Invalid value for formid. It must be a" - " positive integer.") + int_or_parse_error( + xform, "Invalid value for formid. It must be a positive integer." + ) self.xform = get_object_or_404(XForm, pk=xform) xform_qs = XForm.objects.filter(pk=self.xform.pk) - public_forms = XForm.objects.filter(pk=self.xform.pk, - shared_data=True) + public_forms = XForm.objects.filter(pk=self.xform.pk, shared_data=True) else: xform_qs = XForm.objects.all() xform_qs = xform_qs.filter(deleted_at=None) @@ -270,34 +338,35 @@ def _xform_filter(self, request, view, keyword): if request.user.is_anonymous: xforms = xform_qs.filter(shared_data=True) else: - xforms = super(XFormPermissionFilterMixin, self).filter_queryset( - request, xform_qs, view) | public_forms - return {"%s__in" % keyword: xforms} + xforms = super().filter_queryset(request, xform_qs, view) | public_forms + return {f"{keyword}__in": xforms} def _xform_filter_queryset(self, request, queryset, view, keyword): kwarg = self._xform_filter(request, view, keyword) return queryset.filter(**kwarg) -class ProjectPermissionFilterMixin(object): +# pylint: disable=too-few-public-methods +class ProjectPermissionFilterMixin: + """Project permission filter.""" def _project_filter(self, request, view, keyword): project_id = request.query_params.get("project") if project_id: - int_or_parse_error(project_id, - u"Invalid value for projectid. It must be a" - " positive integer.") + int_or_parse_error( + project_id, + "Invalid value for projectid. It must be a positive integer.", + ) project = get_object_or_404(Project, pk=project_id) project_qs = Project.objects.filter(pk=project.id) else: project_qs = Project.objects.all() - projects = super(ProjectPermissionFilterMixin, self).filter_queryset( - request, project_qs, view) + projects = super().filter_queryset(request, project_qs, view) - return {"%s__in" % keyword: projects} + return {f"{keyword}__in": projects} def _project_filter_queryset(self, request, queryset, view, keyword): """Use Project Permissions""" @@ -306,8 +375,11 @@ def _project_filter_queryset(self, request, queryset, view, keyword): return queryset.filter(**kwarg) -class InstancePermissionFilterMixin(object): +# pylint: disable=too-few-public-methods +class InstancePermissionFilterMixin: + """Instance permission filter.""" + # pylint: disable=too-many-locals def _instance_filter(self, request, view, keyword): instance_kwarg = {} instance_content_type = ContentType.objects.get_for_model(Instance) @@ -315,13 +387,14 @@ def _instance_filter(self, request, view, keyword): instance_id = request.query_params.get("instance") project_id = request.query_params.get("project") - xform_id = request.query_params.get('xform') + xform_id = request.query_params.get("xform") if instance_id and project_id and xform_id: for object_id in [instance_id, project_id]: - int_or_parse_error(object_id, - u"Invalid value for instanceid. It must be" - " a positive integer.") + int_or_parse_error( + object_id, + "Invalid value for instanceid. It must be" " a positive integer.", + ) instance = get_object_or_404(Instance, pk=instance_id) # test if user has permissions on the project @@ -337,21 +410,17 @@ def _instance_filter(self, request, view, keyword): project_qs = Project.objects.filter(pk=project.id) if parent and parent.project == project: - projects = super( - InstancePermissionFilterMixin, self).filter_queryset( - request, project_qs, view) + projects = super().filter_queryset(request, project_qs, view) instances = [instance.id] if projects else [] - instance_kwarg["%s__in" % keyword] = instances + instance_kwarg[f"{keyword}__in"] = instances return instance_kwarg - else: - return {} + return {} - else: - return instance_kwarg + return instance_kwarg def _instance_filter_queryset(self, request, queryset, view, keyword): kwarg = self._instance_filter(request, view, keyword) @@ -359,23 +428,27 @@ def _instance_filter_queryset(self, request, queryset, view, keyword): return queryset.filter(**kwarg) -class RestServiceFilter(XFormPermissionFilterMixin, - ObjectPermissionsFilter): +# pylint: disable=too-few-public-methods +class RestServiceFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): + """Rest service filter.""" def filter_queryset(self, request, queryset, view): - return self._xform_filter_queryset( - request, queryset, view, 'xform_id') + return self._xform_filter_queryset(request, queryset, view, "xform_id") -class MetaDataFilter(ProjectPermissionFilterMixin, - InstancePermissionFilterMixin, - XFormPermissionFilterMixin, - ObjectPermissionsFilter): +# pylint: disable=too-few-public-methods +class MetaDataFilter( + ProjectPermissionFilterMixin, + InstancePermissionFilterMixin, + XFormPermissionFilterMixin, + ObjectPermissionsFilter, +): + """Meta data filter.""" def filter_queryset(self, request, queryset, view): keyword = "object_id" - xform_id = request.query_params.get('xform') + xform_id = request.query_params.get("xform") project_id = request.query_params.get("project") instance_id = request.query_params.get("instance") @@ -392,52 +465,61 @@ def filter_queryset(self, request, queryset, view): # return instance specific metadata if instance_id: - return (queryset.filter(Q(**instance_kwarg)) - if (xform_id and instance_kwarg) else []) - elif xform_id: + return ( + queryset.filter(Q(**instance_kwarg)) + if (xform_id and instance_kwarg) + else [] + ) + if xform_id: # return xform specific metadata return queryset.filter(Q(**xform_kwarg)) # return project specific metadata - elif project_id: + if project_id: return queryset.filter(Q(**project_kwarg)) # return all project,instance and xform metadata information - return queryset.filter(Q(**xform_kwarg) | Q(**project_kwarg) | - Q(**instance_kwarg)) + return queryset.filter( + Q(**xform_kwarg) | Q(**project_kwarg) | Q(**instance_kwarg) + ) -class AttachmentFilter(XFormPermissionFilterMixin, - ObjectPermissionsFilter): +# pylint: disable=too-few-public-methods +class AttachmentFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): + """Attachment filter.""" def filter_queryset(self, request, queryset, view): - queryset = self._xform_filter_queryset(request, queryset, view, - 'instance__xform') + queryset = self._xform_filter_queryset( + request, queryset, view, "instance__xform" + ) # Ensure queryset is filtered by XForm meta permissions - xform_ids = set( - queryset.values_list("instance__xform", flat=True)) + xform_ids = set(queryset.values_list("instance__xform", flat=True)) for xform_id in xform_ids: xform = XForm.objects.get(id=xform_id) user = request.user queryset = exclude_items_from_queryset_using_xform_meta_perms( - xform, user, queryset) + xform, user, queryset + ) - instance_id = request.query_params.get('instance') + instance_id = request.query_params.get("instance") if instance_id: - int_or_parse_error(instance_id, - u"Invalid value for instance_id. It must be" - " a positive integer.") + int_or_parse_error( + instance_id, + "Invalid value for instance_id. It must be" " a positive integer.", + ) instance = get_object_or_404(Instance, pk=instance_id) queryset = queryset.filter(instance=instance) return queryset +# pylint: disable=too-few-public-methods class AttachmentTypeFilter(filters.BaseFilterBackend): + """Attachment type filter using `type` query parameter.""" def filter_queryset(self, request, queryset, view): - attachment_type = request.query_params.get('type') + attachment_type = request.query_params.get("type") mime_types = MEDIA_FILE_TYPES.get(attachment_type) @@ -447,35 +529,41 @@ def filter_queryset(self, request, queryset, view): return queryset +# pylint: disable=too-few-public-methods class TeamOrgFilter(filters.BaseFilterBackend): + """Team organization filter using `org` query parameter""" def filter_queryset(self, request, queryset, view): - org = request.data.get('org') or request.query_params.get('org') + org = request.data.get("org") or request.query_params.get("org") # Get all the teams for the organization if org: - kwargs = { - 'organization__username__iexact': org - } + kwargs = {"organization__username__iexact": org} return Team.objects.filter(**kwargs) return queryset +# pylint: disable=too-few-public-methods class UserNoOrganizationsFilter(filters.BaseFilterBackend): + """Filter by ``orgs`` query parameter.""" def filter_queryset(self, request, queryset, view): - if str(request.query_params.get('orgs')).lower() == 'false': + """Returns all users that are not organizations when `orgs=false` + query parameter""" + if str(request.query_params.get("orgs")).lower() == "false": organization_user_ids = OrganizationProfile.objects.values_list( - 'user__id', - flat=True) + "user__id", flat=True + ) queryset = queryset.exclude(id__in=organization_user_ids) return queryset +# pylint: disable=too-few-public-methods class OrganizationsSharedWithUserFilter(filters.BaseFilterBackend): + """Filters by ``shared_with`` query parameter.""" def filter_queryset(self, request, queryset, view): """ @@ -483,7 +571,7 @@ def filter_queryset(self, request, queryset, view): the passed user belongs. """ - username = request.query_params.get('shared_with') + username = request.query_params.get("shared_with") if username: try: @@ -491,49 +579,50 @@ def filter_queryset(self, request, queryset, view): # Groups a User belongs to are available as a queryset property # of a User object, which this code takes advantage of - organization_user_ids = User.objects\ - .get(username=username)\ - .groups\ - .all()\ - .values_list( - 'team__organization', - flat=True)\ - .distinct() + organization_user_ids = ( + User.objects.get(username=username) + .groups.all() + .values_list("team__organization", flat=True) + .distinct() + ) - filtered_queryset = queryset.filter( - user_id__in=organization_user_ids) + filtered_queryset = queryset.filter(user_id__in=organization_user_ids) return filtered_queryset - except ObjectDoesNotExist: - raise Http404 + except ObjectDoesNotExist as non_existent_object: + raise Http404 from non_existent_object return queryset -class WidgetFilter(XFormPermissionFilterMixin, - ObjectPermissionsFilter): +# pylint: disable=too-few-public-methods +class WidgetFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): + """Filter to return forms shared with user.""" def filter_queryset(self, request, queryset, view): + """Filter to return forms shared with user when ``view.action == "list"``.""" - if view.action == 'list': + if view.action == "list": # Return widgets from xform user has perms to - return self._xform_filter_queryset(request, queryset, view, - 'object_id') + return self._xform_filter_queryset(request, queryset, view, "object_id") - return super(WidgetFilter, self).filter_queryset(request, queryset, - view) + return super().filter_queryset(request, queryset, view) +# pylint: disable=too-few-public-methods class UserProfileFilter(filters.BaseFilterBackend): + """Filter by the ``users`` query parameter.""" def filter_queryset(self, request, queryset, view): - if view.action == 'list': - users = request.GET.get('users') + """Filter by the ``users`` query parameter - returns a queryset of only the users + in the users parameter when `view.action == "list"`""" + if view.action == "list": + users = request.GET.get("users") if users: - users = users.split(',') + users = users.split(",") return queryset.filter(user__username__in=users) - elif not request.user.is_anonymous: + if not request.user.is_anonymous: return queryset.filter(user__username=request.user.username) return queryset.none() @@ -541,14 +630,19 @@ def filter_queryset(self, request, queryset, view): return queryset +# pylint: disable=too-few-public-methods class NoteFilter(filters.BaseFilterBackend): + """Notes filter by the query parameter ``instance``.""" + def filter_queryset(self, request, queryset, view): - instance_id = request.query_params.get('instance') + """Notes filter by the query parameter ``instance``.""" + instance_id = request.query_params.get("instance") if instance_id: - int_or_parse_error(instance_id, - u"Invalid value for instance_id. It must be" - " a positive integer") + int_or_parse_error( + instance_id, + "Invalid value for instance_id. It must be" " a positive integer", + ) instance = get_object_or_404(Instance, pk=instance_id) queryset = queryset.filter(instance=instance) @@ -556,51 +650,51 @@ def filter_queryset(self, request, queryset, view): return queryset -class ExportFilter(XFormPermissionFilterMixin, - ObjectPermissionsFilter): +# pylint: disable=too-few-public-methods +class ExportFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): """ ExportFilter class uses permissions on the related xform to filter Export - queryesets. Also filters submitted_by a specific user. + querysets. Also filters submitted_by a specific user. """ - def _is_public_xform(self, export_id: int): - export = Export.objects.filter(pk=export_id).first() - - if export: - return export.xform.shared_data or export.xform.shared - - return False - def filter_queryset(self, request, queryset, view): - has_submitted_by_key = (Q(options__has_key='query') & - Q(options__query__has_key='_submitted_by'),) + """Filter by xform permissions and submitted by user.""" + has_submitted_by_key = ( + Q(options__has_key="query") & Q(options__query__has_key="_submitted_by"), + ) - if request.user.is_anonymous or self._is_public_xform( - view.kwargs.get('pk')): + if request.user.is_anonymous or _is_public_xform(view.kwargs.get("pk")): return self._xform_filter_queryset( - request, queryset, view, 'xform_id')\ - .exclude(*has_submitted_by_key) + request, queryset, view, "xform_id" + ).exclude(*has_submitted_by_key) - old_perm_format = self.perm_format + old_perm_format = getattr(self, "perm_format") # only if request.user has access to all data - self.perm_format = old_perm_format + '_all' - all_qs = self._xform_filter_queryset(request, queryset, view, - 'xform_id')\ - .exclude(*has_submitted_by_key) + # noqa pylint: disable=attribute-defined-outside-init + self.perm_format = old_perm_format + "_all" + all_qs = self._xform_filter_queryset( + request, queryset, view, "xform_id" + ).exclude(*has_submitted_by_key) # request.user has access to own submitted data - self.perm_format = old_perm_format + '_data' - submitter_qs = self._xform_filter_queryset(request, queryset, view, - 'xform_id')\ - .filter(*has_submitted_by_key)\ + self.perm_format = old_perm_format + "_data" + submitter_qs = ( + self._xform_filter_queryset(request, queryset, view, "xform_id") + .filter(*has_submitted_by_key) .filter(options__query___submitted_by=request.user.username) + ) return all_qs | submitter_qs -class PublicDatasetsFilter(object): +# pylint: disable=too-few-public-methods +class PublicDatasetsFilter: + """Public data set filter where the share attribute is True""" + + # pylint: disable=unused-argument,no-self-use def filter_queryset(self, request, queryset, view): + """Return a queryset of shared=True data if the user is anonymous.""" if request and request.user.is_anonymous: return queryset.filter(shared=True) diff --git a/onadata/libs/mixins/labels_mixin.py b/onadata/libs/mixins/labels_mixin.py index 99205186f3..836c16672a 100644 --- a/onadata/libs/mixins/labels_mixin.py +++ b/onadata/libs/mixins/labels_mixin.py @@ -1,4 +1,7 @@ +# -*- coding: utf-8 -*- +"""LabelMixin module""" from django import forms + from rest_framework import status from rest_framework.decorators import action from rest_framework.response import Response @@ -9,6 +12,8 @@ class TagForm(forms.Form): + """TagForm form""" + tags = TagField() @@ -22,7 +27,7 @@ def _labels_post(request, instance): form = TagForm(request.data) if form.is_valid(): - tags = form.cleaned_data.get('tags', None) + tags = form.cleaned_data.get("tags", None) if tags: for tag in tags: @@ -32,6 +37,7 @@ def _labels_post(request, instance): xform_tags_add.send(sender=XForm, xform=instance, tags=tags) return status.HTTP_201_CREATED + return None def _labels_delete(label, instance): @@ -49,8 +55,11 @@ def _labels_delete(label, instance): xform_tags_delete.send(sender=XForm, xform=instance, tag=label) # Accepted, label does not exist hence nothing removed - http_status = status.HTTP_202_ACCEPTED \ - if count == instance.tags.names().count() else status.HTTP_200_OK + http_status = ( + status.HTTP_202_ACCEPTED + if count == instance.tags.names().count() + else status.HTTP_200_OK + ) return [http_status, list(instance.tags.names())] @@ -66,13 +75,12 @@ def process_label_request(request, label, instance): """ http_status = status.HTTP_200_OK - if request.method == 'POST': + if request.method == "POST": http_status = _labels_post(request, instance) - if request.method == 'GET' and label: - data = [tag['name'] - for tag in instance.tags.filter(name=label).values('name')] - elif request.method == 'DELETE' and label: + if request.method == "GET" and label: + data = [tag["name"] for tag in instance.tags.filter(name=label).values("name")] + elif request.method == "DELETE" and label: http_status, data = _labels_delete(label, instance) else: data = list(instance.tags.names()) @@ -80,11 +88,26 @@ def process_label_request(request, label, instance): return Response(data, status=http_status) -class LabelsMixin(object): - @action(methods=['GET', 'POST', 'DELETE'], detail=True, - extra_lookup_fields=['label', ]) - def labels(self, request, format='json', **kwargs): +# pylint: disable=too-few-public-methods +class LabelsMixin: + """LabelsMixin - adds labels method that processes labels.""" + + # pylint: disable=redefined-builtin + @action( + methods=["GET", "POST", "DELETE"], + detail=True, + extra_lookup_fields=[ + "label", + ], + ) + def labels(self, request, **kwargs): + """Process request to labels endpoint. + + :param request: HTTP request object. + + :returns: A response object based on the type of request. + """ xform = self.get_object() - label = kwargs.get('label') + label = kwargs.get("label") return process_label_request(request, label, xform) diff --git a/onadata/libs/models/clone_xform.py b/onadata/libs/models/clone_xform.py index 467c6b8539..d618837eb3 100644 --- a/onadata/libs/models/clone_xform.py +++ b/onadata/libs/models/clone_xform.py @@ -1,33 +1,40 @@ -from django.contrib.auth.models import User -from onadata.apps.viewer.models.data_dictionary import \ - DataDictionary, upload_to +# -*- coding: utf-8 -*- +""" +CloneXForm class model. +""" +from django.contrib.auth import get_user_model from django.core.files.storage import default_storage + from onadata.apps.logger.models.xform import XForm +from onadata.apps.viewer.models.data_dictionary import DataDictionary, upload_to from onadata.libs.utils.user_auth import get_user_default_project -class CloneXForm(object): +class CloneXForm: + """The class takes an existing form's XLSForm and publishes it as a new form.""" + def __init__(self, xform, username, project=None): self.xform = xform self.username = username self.project = project + self.cloned_form = None @property def user(self): - return User.objects.get(username=self.username) + """Returns a User object for the given ``self.username``.""" + return get_user_model().objects.get(username=self.username) def save(self, **kwargs): - user = User.objects.get(username=self.username) + """Publishes an exiting form's XLSForm as a new form.""" + user = self.user project = self.project or get_user_default_project(user) - xls_file_path = upload_to(None, '%s%s.xlsx' % ( - self.xform.id_string, - XForm.CLONED_SUFFIX), - self.username) + xls_file_path = upload_to( + None, + f"{self.xform.id_string}{XForm.CLONED_SUFFIX}.xlsx", + self.username, + ) xls_data = default_storage.open(self.xform.xls.name) xls_file = default_storage.save(xls_file_path, xls_data) self.cloned_form = DataDictionary.objects.create( - user=user, - created_by=user, - xls=xls_file, - project=project + user=user, created_by=user, xls=xls_file, project=project ) diff --git a/onadata/libs/models/share_project.py b/onadata/libs/models/share_project.py index 7cea9ded74..5ea246cf57 100644 --- a/onadata/libs/models/share_project.py +++ b/onadata/libs/models/share_project.py @@ -1,25 +1,45 @@ -from django.contrib.auth.models import User +# -*- coding: utf-8 -*- +""" +ShareProject model - facilitate sharing of a project to a user. +""" +from django.contrib.auth import get_user_model from django.db import transaction from onadata.libs.permissions import ROLES -from onadata.libs.permissions import EditorRole, EditorMinorRole,\ - DataEntryRole, DataEntryMinorRole, DataEntryOnlyRole +from onadata.libs.permissions import ( + EditorRole, + EditorMinorRole, + DataEntryRole, + DataEntryMinorRole, + DataEntryOnlyRole, +) from onadata.libs.utils.cache_tools import ( - PROJ_PERM_CACHE, PROJ_OWNER_CACHE, safe_delete) + PROJ_PERM_CACHE, + PROJ_OWNER_CACHE, + safe_delete, +) + +# pylint: disable=invalid-name +User = get_user_model() def remove_xform_permissions(project, user, role): + """Remove user permissions to all forms for the given ``project``.""" # remove role from project forms as well for xform in project.xform_set.all(): + # pylint: disable=protected-access role._remove_obj_permissions(user, xform) def remove_dataview_permissions(project, user, role): + """Remove user permissions to all dataviews for the given ``project``.""" for dataview in project.dataview_set.all(): + # pylint: disable=protected-access role._remove_obj_permissions(user, dataview.xform) -class ShareProject(object): +class ShareProject: + """Share project with a user.""" def __init__(self, project, username, role, remove=False): self.project = project @@ -29,11 +49,14 @@ def __init__(self, project, username, role, remove=False): @property def user(self): + """Return the user object for the given ``self.username``.""" return User.objects.get(username=self.username) + # pylint: disable=unused-argument @transaction.atomic() def save(self, **kwargs): - + """Assigns role permissions to a project for the user.""" + # pylint: disable=too-many-nested-blocks if self.remove: self.__remove_user() else: @@ -45,8 +68,7 @@ def save(self, **kwargs): # apply same role to forms under the project for xform in self.project.xform_set.all(): # check if there is xform meta perms set - meta_perms = xform.metadata_set \ - .filter(data_type='xform_meta_perms') + meta_perms = xform.metadata_set.filter(data_type="xform_meta_perms") if meta_perms: meta_perm = meta_perms[0].data_value.split("|") @@ -54,8 +76,11 @@ def save(self, **kwargs): if role in [EditorRole, EditorMinorRole]: role = ROLES.get(meta_perm[0]) - elif role in [DataEntryRole, DataEntryMinorRole, - DataEntryOnlyRole]: + elif role in [ + DataEntryRole, + DataEntryMinorRole, + DataEntryOnlyRole, + ]: role = ROLES.get(meta_perm[1]) role.add(self.user, xform) @@ -64,8 +89,8 @@ def save(self, **kwargs): role.add(self.user, dataview.xform) # clear cache - safe_delete('{}{}'.format(PROJ_OWNER_CACHE, self.project.pk)) - safe_delete('{}{}'.format(PROJ_PERM_CACHE, self.project.pk)) + safe_delete(f"{PROJ_OWNER_CACHE}{self.project.pk}") + safe_delete(f"{PROJ_PERM_CACHE}{self.project.pk}") @transaction.atomic() def __remove_user(self): @@ -74,4 +99,5 @@ def __remove_user(self): if role and self.user and self.project: remove_xform_permissions(self.project, self.user, role) remove_dataview_permissions(self.project, self.user, role) + # pylint: disable=protected-access role._remove_obj_permissions(self.user, self.project) diff --git a/onadata/libs/models/share_team_project.py b/onadata/libs/models/share_team_project.py index 675d6fa1f2..e4b8292402 100644 --- a/onadata/libs/models/share_team_project.py +++ b/onadata/libs/models/share_team_project.py @@ -1,56 +1,77 @@ -from onadata.libs.permissions import DataEntryRole, DataEntryMinorRole, \ - DataEntryOnlyRole, EditorMinorRole, EditorRole, ROLES +# -*- coding: utf-8 -*- +""" +ShareTeamProject model - facilitate sharing a project to a team. +""" +from onadata.libs.permissions import ( + ROLES, + DataEntryMinorRole, + DataEntryOnlyRole, + DataEntryRole, + EditorMinorRole, + EditorRole, +) from onadata.libs.utils.cache_tools import PROJ_PERM_CACHE, safe_delete from onadata.libs.utils.common_tags import XFORM_META_PERMS -class ShareTeamProject(object): +class ShareTeamProject: + """Share a project to a team for the given role.""" + def __init__(self, team, project, role, remove=False): self.team = team self.project = project self.role = role self.remove = remove + # pylint: disable=unused-argument def save(self, **kwargs): + """Assigns project role permissions to the team.""" + # pylint: disable=too-many-nested-blocks if self.remove: - return self.remove_team() - - role = ROLES.get(self.role) + self.remove_team() + else: + role = ROLES.get(self.role) - if role and self.team and self.project: - role.add(self.team, self.project) + if role and self.team and self.project: + role.add(self.team, self.project) - for xform in self.project.xform_set.all(): - # check if there is xform meta perms set - meta_perms = xform.metadata_set \ - .filter(data_type=XFORM_META_PERMS) - if meta_perms: - meta_perm = meta_perms[0].data_value.split("|") + for xform in self.project.xform_set.all(): + # check if there is xform meta perms set + meta_perms = xform.metadata_set.filter(data_type=XFORM_META_PERMS) + if meta_perms: + meta_perm = meta_perms[0].data_value.split("|") - if len(meta_perm) > 1: - if role in [EditorRole, EditorMinorRole]: - role = ROLES.get(meta_perm[0]) + if len(meta_perm) > 1: + if role in [EditorRole, EditorMinorRole]: + role = ROLES.get(meta_perm[0]) - elif role in [DataEntryRole, DataEntryMinorRole, - DataEntryOnlyRole]: - role = ROLES.get(meta_perm[1]) - role.add(self.team, xform) + elif role in [ + DataEntryRole, + DataEntryMinorRole, + DataEntryOnlyRole, + ]: + role = ROLES.get(meta_perm[1]) + role.add(self.team, xform) - for dataview in self.project.dataview_set.all(): - if dataview.matches_parent: - role.add(self.user, dataview.xform) + for dataview in self.project.dataview_set.all(): + if dataview.matches_parent: + role.add(self.team, dataview.xform) - # clear cache - safe_delete('{}{}'.format(PROJ_PERM_CACHE, self.project.pk)) + # clear cache + safe_delete(f"{PROJ_PERM_CACHE}{self.project.pk}") def remove_team(self): + """Removes team permissions from a project.""" role = ROLES.get(self.role) if role and self.team and self.project: + # pylint: disable=protected-access role._remove_obj_permissions(self.team, self.project) for xform in self.project.xform_set.all(): + # pylint: disable=protected-access role._remove_obj_permissions(self.team, xform) for dataview in self.project.dataview_set.all(): + # pylint: disable=protected-access role._remove_obj_permissions(self.team, dataview.xform) diff --git a/onadata/libs/models/share_xform.py b/onadata/libs/models/share_xform.py index 10e8214341..99d75b6b97 100644 --- a/onadata/libs/models/share_xform.py +++ b/onadata/libs/models/share_xform.py @@ -1,10 +1,22 @@ -from django.contrib.auth.models import User -from onadata.libs.permissions import ROLES -from onadata.libs.permissions import EditorRole, EditorMinorRole,\ - DataEntryRole, DataEntryMinorRole, DataEntryOnlyRole +# -*- coding: utf-8 -*- +""" +ShareXForm model - facilitates sharing a form. +""" +from django.contrib.auth import get_user_model +from onadata.libs.permissions import ( + ROLES, + DataEntryMinorRole, + DataEntryOnlyRole, + DataEntryRole, + EditorMinorRole, + EditorRole, +) + + +class ShareXForm: + """ShareXForm class to facilitate sharing a form to a user with specified role.""" -class ShareXForm(object): def __init__(self, xform, username, role): self.xform = xform self.username = username @@ -12,14 +24,16 @@ def __init__(self, xform, username, role): @property def user(self): - return User.objects.get(username=self.username) + """Returns the user object matching ``self.username``.""" + return get_user_model().objects.get(username=self.username) + # pylint: disable=unused-argument def save(self, **kwargs): + """Assign specified role permission to a user for the given form.""" role = ROLES.get(self.role) # # check if there is xform meta perms set - meta_perms = self.xform.metadata_set\ - .filter(data_type='xform_meta_perms') + meta_perms = self.xform.metadata_set.filter(data_type="xform_meta_perms") if meta_perms: meta_perm = meta_perms[0].data_value.split("|") @@ -27,8 +41,7 @@ def save(self, **kwargs): if role in [EditorRole, EditorMinorRole]: role = ROLES.get(meta_perm[0]) - elif role in [DataEntryRole, DataEntryMinorRole, - DataEntryOnlyRole]: + elif role in [DataEntryRole, DataEntryMinorRole, DataEntryOnlyRole]: role = ROLES.get(meta_perm[1]) if role and self.user and self.xform: diff --git a/onadata/libs/models/signals.py b/onadata/libs/models/signals.py index 63992c77aa..5c395078a2 100644 --- a/onadata/libs/models/signals.py +++ b/onadata/libs/models/signals.py @@ -1,17 +1,20 @@ -from past.builtins import basestring - +# -*- coding: utf-8 -*- +"""onadata.libs.models.signals module""" import django.dispatch from onadata.apps.logger.models import XForm -xform_tags_add = django.dispatch.Signal(providing_args=['xform', 'tags']) -xform_tags_delete = django.dispatch.Signal(providing_args=['xform', 'tag']) +# pylint: disable=unexpected-keyword-arg +xform_tags_add = django.dispatch.Signal(providing_args=["xform", "tags"]) +xform_tags_delete = django.dispatch.Signal(providing_args=["xform", "tag"]) +# pylint: disable=unused-argument @django.dispatch.receiver(xform_tags_add, sender=XForm) def add_tags_to_xform_instances(sender, **kwargs): - xform = kwargs.get('xform', None) - tags = kwargs.get('tags', None) + """Adds tags to an xform instance.""" + xform = kwargs.get("xform", None) + tags = kwargs.get("tags", None) if isinstance(xform, XForm) and isinstance(tags, list): # update existing instances with the new tag for instance in xform.instances.all(): @@ -22,11 +25,13 @@ def add_tags_to_xform_instances(sender, **kwargs): instance.parsed_instance.save() +# pylint: disable=unused-argument @django.dispatch.receiver(xform_tags_delete, sender=XForm) def delete_tag_from_xform_instances(sender, **kwargs): - xform = kwargs.get('xform', None) - tag = kwargs.get('tag', None) - if isinstance(xform, XForm) and isinstance(tag, basestring): + """Deletes tags associated with an xform when it is deleted.""" + xform = kwargs.get("xform", None) + tag = kwargs.get("tag", None) + if isinstance(xform, XForm) and isinstance(tag, str): # update existing instances with the new tag for instance in xform.instances.all(): if tag in instance.tags.names(): diff --git a/onadata/libs/models/textit_service.py b/onadata/libs/models/textit_service.py index 993062b870..a5bc4492ed 100644 --- a/onadata/libs/models/textit_service.py +++ b/onadata/libs/models/textit_service.py @@ -3,28 +3,38 @@ TextItService model: sets up all properties for interaction with TextIt or RapidPro. """ -from onadata.apps.main.models.meta_data import MetaData -from onadata.apps.restservice.models import RestService from django.conf import settings -from django.utils.translation import ugettext as _ from django.db import IntegrityError +from django.utils.translation import gettext as _ + from rest_framework import serializers +from onadata.apps.main.models.meta_data import MetaData +from onadata.apps.restservice.models import RestService + METADATA_SEPARATOR = settings.METADATA_SEPARATOR -# pylint: disable=R0902 -class TextItService(object): +# pylint: disable=too-many-instance-attributes +class TextItService: """ TextItService model: access/create/update RestService and MetaData objects with all properties for TextIt or RapidPro like services. """ - # pylint: disable=R0913 - def __init__(self, xform, service_url=None, name=None, auth_token=None, - flow_uuid=None, contacts=None, - pk=None, flow_title: str = ""): - self.pk = pk # pylint: disable=C0103 + # pylint: disable=too-many-arguments,invalid-name + def __init__( + self, + xform, + service_url=None, + name=None, + auth_token=None, + flow_uuid=None, + contacts=None, + pk=None, + flow_title: str = "", + ): + self.pk = pk self.xform = xform self.auth_token = auth_token self.flow_uuid = flow_uuid @@ -42,8 +52,7 @@ def save(self): Creates and updates RestService and MetaData objects with textit or rapidpro service properties. """ - service = RestService() if not self.pk else \ - RestService.objects.get(pk=self.pk) + service = RestService() if not self.pk else RestService.objects.get(pk=self.pk) service.name = self.name service.service_url = self.service_url @@ -51,9 +60,8 @@ def save(self): try: service.save() except IntegrityError as e: - if str(e).startswith("duplicate key value violates unique " - "constraint"): - msg = _(u"The service already created for this form.") + if str(e).startswith("duplicate key value violates unique constraint"): + msg = _("The service already created for this form.") else: msg = _(str(e)) raise serializers.ValidationError(msg) @@ -64,13 +72,10 @@ def save(self): self.active = service.active self.inactive_reason = service.inactive_reason - data_value = '{}|{}|{}'.format(self.auth_token, - self.flow_uuid, - self.contacts) + data_value = f"{self.auth_token}|{self.flow_uuid}|{self.contacts}" MetaData.textit(self.xform, data_value=data_value) - MetaData.textit_flow_details( - self.xform, data_value=self.flow_title) + MetaData.textit_flow_details(self.xform, data_value=self.flow_title) if self.xform.is_merged_dataset: for xform in self.xform.mergedxform.xforms.all(): @@ -86,11 +91,21 @@ def retrieve(self): """ data_value = MetaData.textit(self.xform) + if data_value is None: + raise serializers.ValidationError( + _( + "Error occurred when loading textit service." + "Resolve by updating auth_token, flow_uuid and contacts fields" + ) + ) try: - self.auth_token, self.flow_uuid, self.contacts = \ - data_value.split(METADATA_SEPARATOR) - except ValueError: + self.auth_token, self.flow_uuid, self.contacts = data_value.split( + METADATA_SEPARATOR + ) + except ValueError as e: raise serializers.ValidationError( - _("Error occurred when loading textit service." - "Resolve by updating auth_token, flow_uuid and contacts" - " fields")) + _( + "Error occurred when loading textit service." + "Resolve by updating auth_token, flow_uuid and contacts fields" + ) + ) from e diff --git a/onadata/libs/renderers/renderers.py b/onadata/libs/renderers/renderers.py index 297f7de322..8a0604f617 100644 --- a/onadata/libs/renderers/renderers.py +++ b/onadata/libs/renderers/renderers.py @@ -9,34 +9,34 @@ from typing import Tuple import pytz -from django.utils import six, timezone +import six + +from django.utils import timezone from django.utils.dateparse import parse_datetime -from django.utils.encoding import smart_text, force_str +from django.utils.encoding import smart_str, force_str from django.utils.xmlutils import SimplerXMLGenerator -from future.utils import iteritems +from six import iteritems from rest_framework import negotiation -from rest_framework.renderers import (BaseRenderer, JSONRenderer, - StaticHTMLRenderer, TemplateHTMLRenderer) +from rest_framework.renderers import ( + BaseRenderer, + JSONRenderer, + StaticHTMLRenderer, + TemplateHTMLRenderer, +) from rest_framework.utils.encoders import JSONEncoder from rest_framework_xml.renderers import XMLRenderer from onadata.libs.utils.osm import get_combined_osm IGNORE_FIELDS = [ - 'formhub/uuid', - 'meta/contactID', - 'meta/deprecatedID', - 'meta/instanceID', - 'meta/sessionID', + "formhub/uuid", + "meta/contactID", + "meta/deprecatedID", + "meta/instanceID", + "meta/sessionID", ] -FORMLIST_MANDATORY_FIELDS = [ - 'formID', - 'name', - 'version', - 'hash', - 'downloadUrl' -] +FORMLIST_MANDATORY_FIELDS = ["formID", "name", "version", "hash", "downloadUrl"] def pairing(val1, val2): @@ -52,16 +52,19 @@ def floip_rows_list(data): """ Yields a row of FLOIP results data from dict data. """ - _submission_time = pytz.timezone('UTC').localize( - parse_datetime(data['_submission_time'])).isoformat() + _submission_time = ( + pytz.timezone("UTC") + .localize(parse_datetime(data["_submission_time"])) + .isoformat() + ) for i, key in enumerate(data, 1): - if not (key.startswith('_') or key in IGNORE_FIELDS): - instance_id = data['_id'] + if not (key.startswith("_") or key in IGNORE_FIELDS): + instance_id = data["_id"] yield [ _submission_time, # Timestamp int(pairing(instance_id, i)), # Row ID - data.get('meta/contactID', data.get('_submitted_by')), - data.get('meta/sessionID') or data.get('_uuid') or instance_id, + data.get("meta/contactID", data.get("_submitted_by")), + data.get("meta/sessionID") or data.get("_uuid") or instance_id, key, # Question ID data[key], # Response None, # Response Metadata @@ -77,30 +80,55 @@ def floip_list(data): yield i +def _pop_xml_attributes(xml_dictionary: dict) -> Tuple[dict, dict]: + """ + Extracts XML attributes from the ``xml_dictionary``. + """ + ret = xml_dictionary.copy() + attributes = {} + + for key, value in xml_dictionary.items(): + if key.startswith("@"): + attributes.update({key.replace("@", ""): value}) + del ret[key] + + return ret, attributes + + class DecimalEncoder(JSONEncoder): """ JSON DecimalEncoder that returns None for decimal nan json values. """ - def default(self, obj): # pylint: disable=method-hidden + # pylint: disable=method-hidden + def default(self, obj): + """ + JSON DecimalEncoder that returns None for decimal nan json values. + """ # Handle Decimal NaN values if isinstance(obj, decimal.Decimal) and math.isnan(obj): return None return JSONEncoder.default(self, obj) -class XLSRenderer(BaseRenderer): # pylint: disable=R0903 +# pylint: disable=abstract-method,too-few-public-methods +class XLSRenderer(BaseRenderer): """ XLSRenderer - renders .xls spreadsheet documents with application/vnd.openxmlformats. """ - media_type = 'application/vnd.openxmlformats' - format = 'xls' + + media_type = "application/vnd.openxmlformats" + format = "xls" charset = None + # pylint: disable=no-self-use,unused-argument def render(self, data, accepted_media_type=None, renderer_context=None): + """ + Encode ``data`` string to 'utf-8'. + """ if isinstance(data, six.text_type): - return data.encode('utf-8') + return data.encode("utf-8") return data @@ -109,30 +137,34 @@ class XLSXRenderer(XLSRenderer): # pylint: disable=too-few-public-methods XLSRenderer - renders .xlsx spreadsheet documents with application/vnd.openxmlformats. """ - format = 'xlsx' + + format = "xlsx" -class CSVRenderer(BaseRenderer): # pylint: disable=abstract-method, R0903 +# pylint: disable=abstract-method, too-few-public-methods +class CSVRenderer(BaseRenderer): """ XLSRenderer - renders comma separated files (CSV) with text/csv. """ - media_type = 'text/csv' - format = 'csv' - charset = 'utf-8' + media_type = "text/csv" + format = "csv" + charset = "utf-8" -class CSVZIPRenderer(BaseRenderer): # pylint: disable=R0903 + +class CSVZIPRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ CSVZIPRenderer - renders a ZIP file that contains CSV files. """ - media_type = 'application/octet-stream' - format = 'csvzip' + + media_type = "application/octet-stream" + format = "csvzip" charset = None def render(self, data, accepted_media_type=None, renderer_context=None): if isinstance(data, six.text_type): - return data.encode('utf-8') - elif isinstance(data, dict): + return data.encode("utf-8") + if isinstance(data, dict): return json.dumps(data) return data @@ -141,14 +173,15 @@ class SAVZIPRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ SAVZIPRenderer - renders a ZIP file that contains SPSS SAV files. """ - media_type = 'application/octet-stream' - format = 'savzip' + + media_type = "application/octet-stream" + format = "savzip" charset = None def render(self, data, accepted_media_type=None, renderer_context=None): if isinstance(data, six.text_type): - return data.encode('utf-8') - elif isinstance(data, dict): + return data.encode("utf-8") + if isinstance(data, dict): return json.dumps(data) return data @@ -157,9 +190,10 @@ class SurveyRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ SurveyRenderer - renders XML data. """ - media_type = 'application/xml' - format = 'xml' - charset = 'utf-8' + + media_type = "application/xml" + format = "xml" + charset = "utf-8" def render(self, data, accepted_media_type=None, renderer_context=None): return data @@ -169,19 +203,21 @@ class KMLRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ KMLRenderer - renders KML XML data. """ - media_type = 'application/xml' - format = 'kml' - charset = 'utf-8' + + media_type = "application/xml" + format = "kml" + charset = "utf-8" def render(self, data, accepted_media_type=None, renderer_context=None): return data -class GoogleSheetsRenderer(XLSRenderer): # pylint: disable=R0903 +class GoogleSheetsRenderer(XLSRenderer): # pylint: disable=too-few-public-methods """ GoogleSheetsRenderer = Google Sheets excel exports. """ - format = 'gsheets' + + format = "gsheets" class MediaFileContentNegotiation(negotiation.DefaultContentNegotiation): @@ -190,46 +226,46 @@ class MediaFileContentNegotiation(negotiation.DefaultContentNegotiation): matching format. """ - def filter_renderers(self, renderers, format): # pylint: disable=W0622 + # pylint: disable=redefined-builtin,no-self-use + def filter_renderers(self, renderers, format): """ If there is a '.json' style format suffix, filter the renderers so that we only negotiation against those that accept that format. If there is no renderer available, we use MediaFileRenderer. """ - renderers = [ - renderer for renderer in renderers if renderer.format == format - ] + renderers = [renderer for renderer in renderers if renderer.format == format] if not renderers: renderers = [MediaFileRenderer()] return renderers -class MediaFileRenderer(BaseRenderer): # pylint: disable=R0903 +class MediaFileRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ MediaFileRenderer - render binary media files. """ - media_type = '*/*' + + media_type = "*/*" format = None charset = None - render_style = 'binary' + render_style = "binary" def render(self, data, accepted_media_type=None, renderer_context=None): if isinstance(data, six.text_type): - return data.encode('utf-8') + return data.encode("utf-8") return data -class XFormListRenderer(BaseRenderer): # pylint: disable=R0903 +class XFormListRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ Renderer which serializes to XML. """ - media_type = 'text/xml' - format = 'xml' - charset = 'utf-8' - root_node = 'xforms' - element_node = 'xform' + media_type = "text/xml" + format = "xml" + charset = "utf-8" + root_node = "xforms" + element_node = "xform" xmlns = "http://openrosa.org/xforms/xformsList" def render(self, data, accepted_media_type=None, renderer_context=None): @@ -237,15 +273,15 @@ def render(self, data, accepted_media_type=None, renderer_context=None): Renders *obj* into serialized XML. """ if data is None: - return '' - elif isinstance(data, six.string_types): + return "" + if isinstance(data, six.string_types): return data stream = BytesIO() xml = SimplerXMLGenerator(stream, self.charset) xml.startDocument() - xml.startElement(self.root_node, {'xmlns': self.xmlns}) + xml.startElement(self.root_node, {"xmlns": self.xmlns}) self._to_xml(xml, data) @@ -274,63 +310,65 @@ def _to_xml(self, xml, data): pass else: - xml.characters(smart_text(data)) + xml.characters(smart_str(data)) -class XFormManifestRenderer(XFormListRenderer): # pylint: disable=R0903 +# pylint: disable=too-few-public-methods +class XFormManifestRenderer(XFormListRenderer): """ XFormManifestRenderer - render XFormManifest XML. """ + root_node = "manifest" element_node = "mediaFile" xmlns = "http://openrosa.org/xforms/xformsManifest" -class TemplateXMLRenderer(TemplateHTMLRenderer): # pylint: disable=R0903 +# pylint: disable=too-few-public-methods +class TemplateXMLRenderer(TemplateHTMLRenderer): """ TemplateXMLRenderer - Render XML template. """ - format = 'xml' - media_type = 'text/xml' + + format = "xml" + media_type = "text/xml" def render(self, data, accepted_media_type=None, renderer_context=None): renderer_context = renderer_context or {} - response = renderer_context['response'] + response = renderer_context["response"] if response and response.exception: - return XMLRenderer().render(data, accepted_media_type, - renderer_context) + return XMLRenderer().render(data, accepted_media_type, renderer_context) - return super(TemplateXMLRenderer, - self).render(data, accepted_media_type, renderer_context) + return super().render(data, accepted_media_type, renderer_context) class InstanceXMLRenderer(XMLRenderer): """ InstanceXMLRenderer - Renders Instance XML """ - root_tag_name = 'submission-batch' - item_tag_name = 'submission-item' + + root_tag_name = "submission-batch" + item_tag_name = "submission-item" def _get_current_buffer_data(self): - if hasattr(self, 'stream'): + if hasattr(self, "stream"): ret = self.stream.getvalue() self.stream.truncate(0) self.stream.seek(0) return ret + return None def stream_data(self, data, serializer): if data is None: yield "" + # pylint: disable=attribute-defined-outside-init self.stream = StringIO() xml = SimplerXMLGenerator(self.stream, self.charset) xml.startDocument() - xml.startElement( - self.root_tag_name, - {'serverTime': timezone.now().isoformat()} - ) + xml.startElement(self.root_tag_name, {"serverTime": timezone.now().isoformat()}) yield self._get_current_buffer_data() @@ -345,7 +383,7 @@ def stream_data(self, data, serializer): try: next_item = next(data) out = serializer(out).data - out, attributes = self._pop_xml_attributes(out) + out, attributes = _pop_xml_attributes(out) xml.startElement(self.item_tag_name, attributes) self._to_xml(xml, out) xml.endElement(self.item_tag_name) @@ -353,7 +391,7 @@ def stream_data(self, data, serializer): out = next_item except StopIteration: out = serializer(out).data - out, attributes = self._pop_xml_attributes(out) + out, attributes = _pop_xml_attributes(out) xml.startElement(self.item_tag_name, attributes) self._to_xml(xml, out) xml.endElement(self.item_tag_name) @@ -376,9 +414,7 @@ def render(self, data, accepted_media_type=None, renderer_context=None): xml = SimplerXMLGenerator(stream, self.charset) xml.startDocument() - xml.startElement( - self.root_tag_name, - {'serverTime': timezone.now().isoformat()}) + xml.startElement(self.root_tag_name, {"serverTime": timezone.now().isoformat()}) self._to_xml(xml, data) @@ -387,21 +423,10 @@ def render(self, data, accepted_media_type=None, renderer_context=None): return stream.getvalue() - def _pop_xml_attributes(self, xml_dictionary: dict) -> Tuple[dict, dict]: - ret = xml_dictionary.copy() - attributes = {} - - for key, value in xml_dictionary.items(): - if key.startswith('@'): - attributes.update({key.replace('@', ''): value}) - del ret[key] - - return ret, attributes - def _to_xml(self, xml, data): if isinstance(data, (list, tuple)): for item in data: - item, attributes = self._pop_xml_attributes(item) + item, attributes = _pop_xml_attributes(item) xml.startElement(self.item_tag_name, attributes) self._to_xml(xml, item) xml.endElement(self.item_tag_name) @@ -417,7 +442,7 @@ def _to_xml(self, xml, data): xml.endElement(key) elif isinstance(value, dict): - value, attributes = self._pop_xml_attributes(value) + value, attributes = _pop_xml_attributes(value) xml.startElement(key, attributes) self._to_xml(xml, value) xml.endElement(key) @@ -433,39 +458,42 @@ def _to_xml(self, xml, data): xml.characters(force_str(data)) -class StaticXMLRenderer(StaticHTMLRenderer): # pylint: disable=R0903 +class StaticXMLRenderer(StaticHTMLRenderer): # pylint: disable=too-few-public-methods """ StaticXMLRenderer - render static XML document. """ - format = 'xml' - media_type = 'text/xml' + format = "xml" + media_type = "text/xml" -class GeoJsonRenderer(BaseRenderer): # pylint: disable=R0903 + +class GeoJsonRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ GeoJsonRenderer - render .geojson data as json. """ - media_type = 'application/json' - format = 'geojson' - charset = 'utf-8' + + media_type = "application/json" + format = "geojson" + charset = "utf-8" def render(self, data, accepted_media_type=None, renderer_context=None): return json.dumps(data) -class OSMRenderer(BaseRenderer): # pylint: disable=R0903 +class OSMRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ OSMRenderer - render .osm data as XML. """ - media_type = 'text/xml' - format = 'osm' - charset = 'utf-8' + + media_type = "text/xml" + format = "osm" + charset = "utf-8" def render(self, data, accepted_media_type=None, renderer_context=None): # Process error before making a list if isinstance(data, dict): - if 'detail' in data: - return u'' + data['detail'] + '' + if "detail" in data: + return "" + data["detail"] + "" # Combine/concatenate the list of osm files to one file def _list(list_or_item): @@ -479,48 +507,52 @@ def _list(list_or_item): return get_combined_osm(data) -class OSMExportRenderer(BaseRenderer): # pylint: disable=R0903, W0223 +# pylint: disable=too-few-public-methods,abstract-method +class OSMExportRenderer(BaseRenderer): """ OSMExportRenderer - render .osm data as XML. """ - media_type = 'text/xml' - format = 'osm' - charset = 'utf-8' + media_type = "text/xml" + format = "osm" + charset = "utf-8" -class DebugToolbarRenderer(TemplateHTMLRenderer): # pylint: disable=R0903 + +# pylint: disable=too-few-public-methods +class DebugToolbarRenderer(TemplateHTMLRenderer): """ DebugToolbarRenderer - render .debug as HTML. """ - media_type = 'text/html' - charset = 'utf-8' - format = 'debug' - template_name = 'debug.html' + + media_type = "text/html" + charset = "utf-8" + format = "debug" + template_name = "debug.html" def render(self, data, accepted_media_type=None, renderer_context=None): data = { - 'debug_data': - str( + "debug_data": str( JSONRenderer().render(data, renderer_context=renderer_context), - self.charset) + self.charset, + ) } - return super(DebugToolbarRenderer, self).render( - data, accepted_media_type, renderer_context) + return super().render(data, accepted_media_type, renderer_context) -class ZipRenderer(BaseRenderer): # pylint: disable=R0903 +class ZipRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ ZipRenderer - render .zip files. """ - media_type = 'application/octet-stream' - format = 'zip' + + media_type = "application/octet-stream" + format = "zip" charset = None def render(self, data, accepted_media_type=None, renderer_context=None): if isinstance(data, six.text_type): - return data.encode('utf-8') - elif isinstance(data, dict): + return data.encode("utf-8") + if isinstance(data, dict): return json.dumps(data) return data @@ -529,6 +561,7 @@ class DecimalJSONRenderer(JSONRenderer): """ Extends the default json renderer to handle Decimal('NaN') values """ + encoder_class = DecimalEncoder @@ -536,19 +569,19 @@ class FLOIPRenderer(JSONRenderer): """ FLOIP Results data renderer. """ - media_type = 'application/vnd.org.flowinterop.results+json' - format = 'json' - charset = 'utf-8' + + media_type = "application/vnd.org.flowinterop.results+json" + format = "json" + charset = "utf-8" def render(self, data, accepted_media_type=None, renderer_context=None): - request = renderer_context['request'] - response = renderer_context['response'] + request = renderer_context["request"] + response = renderer_context["response"] results = data - if request.method == 'GET' and response.status_code == 200: + if request.method == "GET" and response.status_code == 200: if isinstance(data, dict): - results = [i for i in floip_rows_list(data)] + results = list(floip_rows_list(data)) else: - results = [i for i in floip_list(data)] + results = list(floip_list(data)) - return super(FLOIPRenderer, self).render(results, accepted_media_type, - renderer_context) + return super().render(results, accepted_media_type, renderer_context) diff --git a/onadata/libs/serializers/attachment_serializer.py b/onadata/libs/serializers/attachment_serializer.py index d59ecdd959..7c676b2fc4 100644 --- a/onadata/libs/serializers/attachment_serializer.py +++ b/onadata/libs/serializers/attachment_serializer.py @@ -1,5 +1,9 @@ -import json -from future.utils import listvalues +# -*- coding: utf-8 -*- +""" +Attachments serializer. +""" + +from six import itervalues from rest_framework import serializers @@ -13,7 +17,7 @@ def dict_key_for_value(_dict, value): """ This function is used to get key by value in a dictionary """ - return list(_dict)[listvalues(_dict).index(value)] + return list(_dict)[list(itervalues(_dict)).index(value)] def get_path(data, question_name, path_list): @@ -25,69 +29,97 @@ def get_path(data, question_name, path_list): :return: an xpath which is a string or None if name cannot be found :rtype: string or None """ - name = data.get('name') + name = data.get("name") if name == question_name: - return '/'.join(path_list) - elif data.get('children') is not None: - for node in data.get('children'): - path_list.append(node.get('name')) + return "/".join(path_list) + if data.get("children") is not None: + for node in data.get("children"): + path_list.append(node.get("name")) path = get_path(node, question_name, path_list) if path is not None: return path - else: - del path_list[len(path_list) - 1] + del path_list[len(path_list) - 1] return None class AttachmentSerializer(serializers.HyperlinkedModelSerializer): - url = serializers.HyperlinkedIdentityField(view_name='attachment-detail', - lookup_field='pk') + """ + Attachments serializer + """ + + url = serializers.HyperlinkedIdentityField( + view_name="attachment-detail", lookup_field="pk" + ) field_xpath = serializers.SerializerMethodField() download_url = serializers.SerializerMethodField() small_download_url = serializers.SerializerMethodField() medium_download_url = serializers.SerializerMethodField() - xform = serializers.ReadOnlyField(source='instance.xform.pk') - instance = serializers.PrimaryKeyRelatedField( - queryset=Instance.objects.all()) - filename = serializers.ReadOnlyField(source='media_file.name') + xform = serializers.ReadOnlyField(source="instance.xform.pk") + instance = serializers.PrimaryKeyRelatedField(queryset=Instance.objects.all()) + filename = serializers.ReadOnlyField(source="media_file.name") class Meta: - fields = ('url', 'filename', 'mimetype', 'field_xpath', 'id', 'xform', - 'instance', 'download_url', 'small_download_url', - 'medium_download_url') + fields = ( + "url", + "filename", + "mimetype", + "field_xpath", + "id", + "xform", + "instance", + "download_url", + "small_download_url", + "medium_download_url", + ) model = Attachment @check_obj def get_download_url(self, obj): - request = self.context.get('request') + """ + Return attachment download url. + """ + request = self.context.get("request") if obj: path = get_attachment_url(obj) return request.build_absolute_uri(path) if request else path + return "" def get_small_download_url(self, obj): - request = self.context.get('request') + """ + Return attachment download url for resized small image. + """ + request = self.context.get("request") - if obj.mimetype.startswith('image'): - path = get_attachment_url(obj, 'small') + if obj.mimetype.startswith("image"): + path = get_attachment_url(obj, "small") return request.build_absolute_uri(path) if request else path + return "" def get_medium_download_url(self, obj): - request = self.context.get('request') + """ + Return attachment download url for resized medium image. + """ + request = self.context.get("request") - if obj.mimetype.startswith('image'): - path = get_attachment_url(obj, 'medium') + if obj.mimetype.startswith("image"): + path = get_attachment_url(obj, "medium") return request.build_absolute_uri(path) if request else path + return "" + # pylint: disable=no-self-use def get_field_xpath(self, obj): + """ + Return question xpath + """ qa_dict = obj.instance.get_dict() if obj.filename not in qa_dict.values(): return None question_name = dict_key_for_value(qa_dict, obj.filename) - data = json.loads(obj.instance.xform.json) + data = obj.instance.xform.json_dict() return get_path(data, question_name, []) diff --git a/onadata/libs/serializers/chart_serializer.py b/onadata/libs/serializers/chart_serializer.py index a35ab22765..3e2f874fb0 100644 --- a/onadata/libs/serializers/chart_serializer.py +++ b/onadata/libs/serializers/chart_serializer.py @@ -1,5 +1,7 @@ -from past.builtins import basestring - +# -*- coding: utf-8 -*- +""" +Chart serializer. +""" from django.http import Http404 from rest_framework import serializers @@ -9,44 +11,54 @@ from onadata.libs.utils.common_tags import INSTANCE_ID +# pylint: disable=too-many-public-methods class ChartSerializer(serializers.HyperlinkedModelSerializer): + """ + Chart serializer + """ + url = serializers.HyperlinkedIdentityField( - view_name='chart-detail', lookup_field='pk') + view_name="chart-detail", lookup_field="pk" + ) class Meta: model = XForm - fields = ('id', 'id_string', 'url') + fields = ("id", "id_string", "url") +# pylint: disable=too-many-public-methods class FieldsChartSerializer(serializers.ModelSerializer): + """ + Generate chart data for the field. + """ class Meta: model = XForm - def to_representation(self, obj): + def to_representation(self, instance): + """ + Generate chart data for a given field in the request query params. + """ data = {} - request = self.context.get('request') + request = self.context.get("request") - if obj is not None: - fields = obj.survey_elements + if instance is not None: + fields = instance.survey_elements if request: - selected_fields = request.query_params.get('fields') + selected_fields = request.query_params.get("fields") - if isinstance(selected_fields, basestring) \ - and selected_fields != 'all': - fields = selected_fields.split(',') - fields = [e for e in obj.survey_elements - if e.name in fields] + if isinstance(selected_fields, str) and selected_fields != "all": + fields = selected_fields.split(",") + fields = [e for e in instance.survey_elements if e.name in fields] if len(fields) == 0: - raise Http404( - "Field %s does not not exist on the form" % fields) + raise Http404(f"Field {fields} does not not exist on the form") for field in fields: if field.name == INSTANCE_ID: continue - field_data = build_chart_data_for_field(obj, field) + field_data = build_chart_data_for_field(instance, field) data[field.name] = field_data return data diff --git a/onadata/libs/serializers/clone_xform_serializer.py b/onadata/libs/serializers/clone_xform_serializer.py index 6822bf2bd5..0392a41342 100644 --- a/onadata/libs/serializers/clone_xform_serializer.py +++ b/onadata/libs/serializers/clone_xform_serializer.py @@ -1,38 +1,53 @@ -from django.contrib.auth.models import User -from django.utils.translation import ugettext as _ +# -*- coding: utf-8 -*- +""" +Clone an XForm serializer. +""" +from django.contrib.auth import get_user_model +from django.utils.translation import gettext as _ from rest_framework import serializers + from onadata.libs.models.clone_xform import CloneXForm -from onadata.libs.serializers.fields.xform_field import XFormField from onadata.libs.serializers.fields.project_field import ProjectField +from onadata.libs.serializers.fields.xform_field import XFormField class CloneXFormSerializer(serializers.Serializer): + """Clone an xform serializer class""" + xform = XFormField() username = serializers.CharField(max_length=255) project = ProjectField(required=False) + # pylint: disable=no-self-use def create(self, validated_data): + """Uses the CloneXForm class to clone/copy an XForm. + + Returns the CloneXForm instance.""" instance = CloneXForm(**validated_data) instance.save() return instance + # pylint: disable=no-self-use def update(self, instance, validated_data): - instance.xform = validated_data.get('xform', instance.xform) - instance.username = validated_data.get('username', instance.username) - instance.project = validated_data.get('project', instance.project) + instance.xform = validated_data.get("xform", instance.xform) + instance.username = validated_data.get("username", instance.username) + instance.project = validated_data.get("project", instance.project) instance.save() return instance + # pylint: disable=no-self-use def validate_username(self, value): """Check that the username exists""" + # pylint: disable=invalid-name + User = get_user_model() # noqa N806 try: User.objects.get(username=value) - except User.DoesNotExist: - raise serializers.ValidationError(_( - u"User '%(value)s' does not exist." % {"value": value} - )) + except User.DoesNotExist as e: + raise serializers.ValidationError( + _(f"User '{value}' does not exist.") + ) from e return value diff --git a/onadata/libs/serializers/data_serializer.py b/onadata/libs/serializers/data_serializer.py index f09ad5eac8..006ef53e9f 100644 --- a/onadata/libs/serializers/data_serializer.py +++ b/onadata/libs/serializers/data_serializer.py @@ -6,7 +6,7 @@ from io import BytesIO from django.shortcuts import get_object_or_404 -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import exceptions, serializers from rest_framework.reverse import reverse diff --git a/onadata/libs/serializers/dataview_serializer.py b/onadata/libs/serializers/dataview_serializer.py index c94359452d..0a0e323a39 100644 --- a/onadata/libs/serializers/dataview_serializer.py +++ b/onadata/libs/serializers/dataview_serializer.py @@ -1,6 +1,6 @@ import datetime -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.core.cache import cache from rest_framework import serializers diff --git a/onadata/libs/serializers/fields/instance_related_field.py b/onadata/libs/serializers/fields/instance_related_field.py index b2b40354b5..76965f908f 100644 --- a/onadata/libs/serializers/fields/instance_related_field.py +++ b/onadata/libs/serializers/fields/instance_related_field.py @@ -11,6 +11,7 @@ class InstanceRelatedField(serializers.RelatedField): """A custom field to represent the content_object generic relationship""" def get_attribute(self, instance): + """Returns instance pk.""" val = get_object_id_by_content_type(instance, Instance) if val: return val @@ -18,10 +19,11 @@ def get_attribute(self, instance): raise SkipField() def to_internal_value(self, data): + """Validates if the instance exists.""" try: return Instance.objects.get(pk=data) - except ValueError: - raise Exception("instance id should be an integer") + except ValueError as e: + raise Exception("instance id should be an integer") from e def to_representation(self, value): """Serialize instance object""" diff --git a/onadata/libs/serializers/fields/json_field.py b/onadata/libs/serializers/fields/json_field.py index 72b9253d1c..d247f84f13 100644 --- a/onadata/libs/serializers/fields/json_field.py +++ b/onadata/libs/serializers/fields/json_field.py @@ -1,27 +1,44 @@ +# -*- coding: utf-8 -*- +""" +A string is represented as valid JSON and is accessible as a dictionary and vis-a-vis. +""" import json -from builtins import str as text -from past.builtins import basestring + from rest_framework import serializers class JsonField(serializers.Field): + """ + Deserialize a string instance containing a JSON document to a Python object. + """ + # pylint: disable=no-self-use def to_representation(self, value): - if isinstance(value, basestring): + """ + Deserialize ``value`` a `str` instance containing a + JSON document to a Python object. + """ + if isinstance(value, str): return json.loads(value) return value - def to_internal_value(self, value): - if isinstance(value, basestring): + # pylint: disable=no-self-use + def to_internal_value(self, data): + """ + Deserialize ``value`` a `str` instance containing a + JSON document to a Python object. + """ + if isinstance(data, str): try: - return json.loads(value) + return json.loads(data) except ValueError as e: # invalid json - raise serializers.ValidationError(text(e)) - return value + raise serializers.ValidationError(str(e)) + return data @classmethod def to_json(cls, data): - if isinstance(data, basestring): + """Returns the JSON string as a dictionary.""" + if isinstance(data, str): return json.loads(data) return data diff --git a/onadata/libs/serializers/fields/organization_field.py b/onadata/libs/serializers/fields/organization_field.py index 7605db7af2..50cd9932f6 100644 --- a/onadata/libs/serializers/fields/organization_field.py +++ b/onadata/libs/serializers/fields/organization_field.py @@ -1,24 +1,37 @@ -from builtins import str as text -from django.utils.translation import ugettext as _ +# -*- coding: utf-8 -*- +""" +OrganizationField serializer field. +""" +from django.utils.translation import gettext as _ + from rest_framework import serializers from onadata.apps.api.models.organization_profile import OrganizationProfile class OrganizationField(serializers.Field): - def to_representation(self, obj): - return obj.pk + """organization serializer field""" + + # pylint: disable=no-self-use + def to_representation(self, value): + """Return the organization pk.""" + return value.pk + # pylint: disable=no-self-use def to_internal_value(self, data): + """Validate the organization exists.""" if data is not None: try: organization = OrganizationProfile.objects.get(pk=data) - except OrganizationProfile.DoesNotExist: - raise serializers.ValidationError(_( - u"Organization with id '%(value)s' does not exist." % - {"value": data} - )) + except OrganizationProfile.DoesNotExist as e: + raise serializers.ValidationError( + _( + "Organization with id '%(value)s' does not exist." + % {"value": data} + ) + ) from e except ValueError as e: - raise serializers.ValidationError(text(e)) + raise serializers.ValidationError(str(e)) from e return organization + return data diff --git a/onadata/libs/serializers/fields/project_field.py b/onadata/libs/serializers/fields/project_field.py index 07febce5f3..e6673ae752 100644 --- a/onadata/libs/serializers/fields/project_field.py +++ b/onadata/libs/serializers/fields/project_field.py @@ -1,24 +1,34 @@ -from builtins import str as text -from django.utils.translation import ugettext as _ +# -*- coding: utf-8 -*- +""" +ProjectField serializer field. +""" +from django.utils.translation import gettext as _ + from rest_framework import serializers from onadata.apps.logger.models.project import Project class ProjectField(serializers.Field): - def to_representation(self, obj): - return obj.pk + """Project field for use with a Project object/instance.""" + + # pylint: disable=no-self-use + def to_representation(self, value): + """Returns the project pk.""" + return value.pk + # pylint: disable=no-self-use def to_internal_value(self, data): + """Validates that a project exists.""" if data is not None: try: project = Project.objects.get(pk=data) - except Project.DoesNotExist: - raise serializers.ValidationError(_( - u"Project with id '%(value)s' does not exist." % - {"value": data} - )) + except Project.DoesNotExist as e: + raise serializers.ValidationError( + _(f"Project with id '{data}' does not exist.") + ) from e except ValueError as e: - raise serializers.ValidationError(text(e)) + raise serializers.ValidationError(str(e)) from e return project + return data diff --git a/onadata/libs/serializers/fields/project_related_field.py b/onadata/libs/serializers/fields/project_related_field.py index 9451fd13f4..921a79d9d2 100644 --- a/onadata/libs/serializers/fields/project_related_field.py +++ b/onadata/libs/serializers/fields/project_related_field.py @@ -21,8 +21,8 @@ def get_attribute(self, instance): def to_internal_value(self, data): try: return Project.objects.get(pk=data) - except ValueError: - raise Exception("project id should be an integer") + except ValueError as e: + raise Exception("project id should be an integer") from e def to_representation(self, value): """Serialize project object""" diff --git a/onadata/libs/serializers/floip_serializer.py b/onadata/libs/serializers/floip_serializer.py index a3477b6b57..6dc9994b59 100644 --- a/onadata/libs/serializers/floip_serializer.py +++ b/onadata/libs/serializers/floip_serializer.py @@ -15,7 +15,7 @@ from django.core.files.uploadedfile import InMemoryUploadedFile from django.db.models import Q from django.shortcuts import get_object_or_404 -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ import six from floip import survey_to_floip_package @@ -26,10 +26,10 @@ from onadata.apps.logger.models import XForm from onadata.libs.utils.logger_tools import dict2xform, safe_create_instance -CONTACT_ID_INDEX = getattr(settings, 'FLOW_RESULTS_CONTACT_ID_INDEX', 2) -SESSION_ID_INDEX = getattr(settings, 'FLOW_RESULTS_SESSION_ID_INDEX', 3) -QUESTION_INDEX = getattr(settings, 'FLOW_RESULTS_QUESTION_INDEX', 4) -ANSWER_INDEX = getattr(settings, 'FLOW_RESULTS_ANSWER_INDEX', 5) +CONTACT_ID_INDEX = getattr(settings, "FLOW_RESULTS_CONTACT_ID_INDEX", 2) +SESSION_ID_INDEX = getattr(settings, "FLOW_RESULTS_SESSION_ID_INDEX", 3) +QUESTION_INDEX = getattr(settings, "FLOW_RESULTS_QUESTION_INDEX", 4) +ANSWER_INDEX = getattr(settings, "FLOW_RESULTS_ANSWER_INDEX", 5) def _get_user(username): @@ -39,21 +39,24 @@ def _get_user(username): def _get_owner(request): - owner = request.data.get('owner') or request.user + owner = request.data.get("owner") or request.user if isinstance(owner, six.string_types): owner_obj = _get_user(owner) if owner_obj is None: - raise ValidationError( - _(u"User with username %s does not exist." % owner)) + raise ValidationError(_("User with username %s does not exist." % owner)) return owner_obj return owner -def parse_responses(responses, session_id_index=SESSION_ID_INDEX, - question_index=QUESTION_INDEX, answer_index=ANSWER_INDEX, - contact_id_index=CONTACT_ID_INDEX): +def parse_responses( + responses, + session_id_index=SESSION_ID_INDEX, + question_index=QUESTION_INDEX, + answer_index=ANSWER_INDEX, + contact_id_index=CONTACT_ID_INDEX, +): """ Returns individual submission for all responses in a flow-results responses package. @@ -65,11 +68,11 @@ def parse_responses(responses, session_id_index=SESSION_ID_INDEX, continue if current_key is None: current_key = row[session_id_index] - if 'meta' not in submission: - submission['meta'] = { - 'instanceID': 'uuid:%s' % current_key, - 'sessionID': current_key, - 'contactID': row[contact_id_index] + if "meta" not in submission: + submission["meta"] = { + "instanceID": "uuid:%s" % current_key, + "sessionID": current_key, + "contactID": row[contact_id_index], } if current_key != row[session_id_index]: yield submission @@ -84,6 +87,7 @@ class ReadOnlyUUIDField(serializers.ReadOnlyField): """ Custom ReadOnlyField for UUID """ + def to_representation(self, obj): # pylint: disable=no-self-use return str(UUID(obj)) @@ -93,33 +97,38 @@ class FloipListSerializer(serializers.HyperlinkedModelSerializer): """ FloipListSerializer class. """ + url = serializers.HyperlinkedIdentityField( - view_name='flow-results-detail', lookup_field='uuid') - id = ReadOnlyUUIDField(source='uuid') # pylint: disable=C0103 - name = serializers.ReadOnlyField(source='id_string') - created = serializers.ReadOnlyField(source='date_created') - modified = serializers.ReadOnlyField(source='date_modified') + view_name="flow-results-detail", lookup_field="uuid" + ) + id = ReadOnlyUUIDField(source="uuid") # pylint: disable=C0103 + name = serializers.ReadOnlyField(source="id_string") + created = serializers.ReadOnlyField(source="date_created") + modified = serializers.ReadOnlyField(source="date_modified") class JSONAPIMeta: # pylint: disable=old-style-class,no-init,R0903 """ JSON API metaclass. """ - resource_name = 'packages' + + resource_name = "packages" class Meta: model = XForm - fields = ('url', 'id', 'name', 'title', 'created', 'modified') + fields = ("url", "id", "name", "title", "created", "modified") class FloipSerializer(serializers.HyperlinkedModelSerializer): """ FloipSerializer class. """ + url = serializers.HyperlinkedIdentityField( - view_name='floip-detail', lookup_field='pk') + view_name="floip-detail", lookup_field="pk" + ) profile = serializers.SerializerMethodField() - created = serializers.ReadOnlyField(source='date_created') - modified = serializers.ReadOnlyField(source='date_modified') + created = serializers.ReadOnlyField(source="date_created") + modified = serializers.ReadOnlyField(source="date_modified") # pylint: disable=invalid-name flow_results_specification_version = serializers.SerializerMethodField() resources = serializers.SerializerMethodField() @@ -128,26 +137,35 @@ class JSONAPIMeta: # pylint: disable=old-style-class,no-init,R0903 """ JSON API metaclass. """ - resource_name = 'packages' + + resource_name = "packages" class Meta: model = XForm - fields = ('url', 'id', 'id_string', 'title', 'profile', 'created', - 'modified', 'flow_results_specification_version', - 'resources') + fields = ( + "url", + "id", + "id_string", + "title", + "profile", + "created", + "modified", + "flow_results_specification_version", + "resources", + ) def get_profile(self, value): # pylint: disable=no-self-use,W0613 """ Returns the data-package profile. """ - return 'data-package' + return "data-package" # pylint: disable=no-self-use,unused-argument def get_flow_results_specification_version(self, value): """ Returns the flow results specification version. """ - return '1.0.0-rc1' + return "1.0.0-rc1" def get_resources(self, value): # pylint: disable=no-self-use,W0613 """ @@ -158,26 +176,27 @@ def get_resources(self, value): # pylint: disable=no-self-use,W0613 def _process_request(self, request, update_instance=None): data = deepcopy(request.data) - if 'profile' in data and data['profile'] == 'flow-results-package': - data['profile'] = 'data-package' - descriptor = BytesIO(json.dumps(data).encode('utf-8')) + if "profile" in data and data["profile"] == "flow-results-package": + data["profile"] = "data-package" + descriptor = BytesIO(json.dumps(data).encode("utf-8")) descriptor.seek(0, os.SEEK_END) floip_file = InMemoryUploadedFile( descriptor, - 'floip_file', - request.data.get('name') + '.json', - 'application/json', + "floip_file", + request.data.get("name") + ".json", + "application/json", descriptor.tell(), - charset=None) + charset=None, + ) kwargs = { - 'user': request.user, - 'post': None, - 'files': {'floip_file': floip_file}, - 'owner': request.user, + "user": request.user, + "post": None, + "files": {"floip_file": floip_file}, + "owner": request.user, } if update_instance: - kwargs['id_string'] = update_instance.id_string - kwargs['project'] = update_instance.project + kwargs["id_string"] = update_instance.id_string + kwargs["project"] = update_instance.project instance = do_publish_xlsform(**kwargs) if isinstance(instance, XForm): return instance @@ -185,27 +204,32 @@ def _process_request(self, request, update_instance=None): raise serializers.ValidationError(instance) def create(self, validated_data): - request = self.context['request'] + request = self.context["request"] return self._process_request(request) def update(self, instance, validated_data): - request = self.context['request'] + request = self.context["request"] return self._process_request(request, instance) def to_representation(self, instance): - request = self.context['request'] + request = self.context["request"] data_id = str(UUID(instance.uuid)) data_url = request.build_absolute_uri( - reverse('flow-results-responses', kwargs={'uuid': data_id})) + reverse("flow-results-responses", kwargs={"uuid": data_id}) + ) package = survey_to_floip_package( - json.loads(instance.json), data_id, instance.date_created, - instance.date_modified, data_url) + instance.json_dict(), + data_id, + instance.date_created, + instance.date_modified, + data_url, + ) data = package.descriptor - if data['profile'] != 'flow-results-package': - data['profile'] = 'flow-results-package' + if data["profile"] != "flow-results-package": + data["profile"] = "flow-results-package" return data @@ -214,6 +238,7 @@ class FlowResultsResponse(object): # pylint: disable=too-few-public-methods """ FLowResultsResponse class to hold a list of submission ids. """ + id = None # pylint: disable=invalid-name responses = [] duplicates = 0 @@ -229,6 +254,7 @@ class FlowResultsResponseSerializer(serializers.Serializer): FlowResultsResponseSerializer for handling publishing of Flow Results Response package. """ + id = serializers.CharField() # pylint: disable=invalid-name responses = serializers.ListField() duplicates = serializers.IntegerField(read_only=True) @@ -237,23 +263,25 @@ class JSONAPIMeta: # pylint: disable=old-style-class,no-init,R0903 """ JSON API metaclass. """ - resource_name = 'responses' + + resource_name = "responses" def create(self, validated_data): duplicates = 0 - request = self.context['request'] - responses = validated_data['responses'] - uuid = UUID(validated_data['id']) + request = self.context["request"] + responses = validated_data["responses"] + uuid = UUID(validated_data["id"]) xform = get_object_or_404( - XForm, - Q(uuid=str(uuid)) | Q(uuid=uuid.hex), - deleted_at__isnull=True) + XForm, Q(uuid=str(uuid)) | Q(uuid=uuid.hex), deleted_at__isnull=True + ) for submission in parse_responses(responses): - xml_file = BytesIO(dict2xform( - submission, xform.id_string, 'data').encode('utf-8')) + xml_file = BytesIO( + dict2xform(submission, xform.id_string, "data").encode("utf-8") + ) error, _instance = safe_create_instance( - request.user.username, xml_file, [], None, request) + request.user.username, xml_file, [], None, request + ) if error and error.status_code != 202: raise serializers.ValidationError(error) if error and error.status_code == 202: diff --git a/onadata/libs/serializers/merged_xform_serializer.py b/onadata/libs/serializers/merged_xform_serializer.py index ae30de281f..556d3af8c7 100644 --- a/onadata/libs/serializers/merged_xform_serializer.py +++ b/onadata/libs/serializers/merged_xform_serializer.py @@ -8,7 +8,7 @@ import uuid from django.db import transaction -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import serializers from pyxform.builder import create_survey_element_from_dict @@ -22,24 +22,28 @@ def _get_fields_set(xform): - return [(element.get_abbreviated_xpath(), element.type) - for element in xform.survey_elements - if element.type not in ['', 'survey']] + return [ + (element.get_abbreviated_xpath(), element.type) + for element in xform.survey_elements + if element.type not in ["", "survey"] + ] def _get_elements(elements, intersect, parent_prefix=None): new_elements = [] for element in elements: - name = element['name'] - name = name if not parent_prefix else '/'.join([parent_prefix, name]) + name = element["name"] + name = name if not parent_prefix else "/".join([parent_prefix, name]) if name in intersect: k = element.copy() - if 'children' in element and element['type'] not in SELECTS: - k['children'] = _get_elements( - element['children'], - [__ for __ in intersect if __.startswith(name)], name) - if not k['children']: + if "children" in element and element["type"] not in SELECTS: + k["children"] = _get_elements( + element["children"], + [__ for __ in intersect if __.startswith(name)], + name, + ) + if not k["children"]: continue new_elements.append(k) @@ -54,44 +58,44 @@ def get_merged_xform_survey(xforms): :param xforms: A list of XForms of at least length 2. """ if len(xforms) < 2: - raise serializers.ValidationError(_('Expecting at least 2 xforms')) + raise serializers.ValidationError(_("Expecting at least 2 xforms")) xform_sets = [_get_fields_set(xform) for xform in xforms] - merged_xform_dict = json.loads(xforms[0].json) - children = merged_xform_dict.pop('children') - merged_xform_dict['children'] = [] + merged_xform_dict = xforms[0].json_dict() + children = merged_xform_dict.pop("children") + merged_xform_dict["children"] = [] intersect = set(xform_sets[0]).intersection(*xform_sets[1:]) intersect = set([__ for (__, ___) in intersect]) - merged_xform_dict['children'] = _get_elements(children, intersect) + merged_xform_dict["children"] = _get_elements(children, intersect) - if '_xpath' in merged_xform_dict: - del merged_xform_dict['_xpath'] + if "_xpath" in merged_xform_dict: + del merged_xform_dict["_xpath"] is_empty = True - xform_dicts = [json.loads(xform.json) for xform in xforms] - for child in merged_xform_dict['children']: - if child['name'] != 'meta' and is_empty: + xform_dicts = [xform.json_dict() for xform in xforms] + for child in merged_xform_dict["children"]: + if child["name"] != "meta" and is_empty: is_empty = False # Remove bind attributes from child elements - if 'bind' in child: - del child['bind'] + if "bind" in child: + del child["bind"] # merge select one and select multiple options - if 'children' in child and child['type'] in SELECTS: + if "children" in child and child["type"] in SELECTS: children = [] for xform_dict in xform_dicts: - element_name = child['name'] + element_name = child["name"] element_list = list( - filter(lambda x: x['name'] == element_name, - xform_dict['children'])) + filter(lambda x: x["name"] == element_name, xform_dict["children"]) + ) if element_list and element_list[0]: - children += element_list[0]['children'] + children += element_list[0]["children"] # remove duplicates set_of_jsons = {json.dumps(d, sort_keys=True) for d in children} - child['children'] = [json.loads(t) for t in set_of_jsons] + child["children"] = [json.loads(t) for t in set_of_jsons] if is_empty: raise serializers.ValidationError(_("No matching fields in xforms.")) @@ -103,11 +107,11 @@ def minimum_two_xforms(value): """Validate at least 2 xforms are provided""" if len(value) < 2: raise serializers.ValidationError( - _('This field should have at least two unique xforms.')) + _("This field should have at least two unique xforms.") + ) if len(set(value)) != len(value): - raise serializers.ValidationError( - _('This field should have unique xforms')) + raise serializers.ValidationError(_("This field should have unique xforms")) return value @@ -125,14 +129,23 @@ class XFormSerializer(serializers.HyperlinkedModelSerializer): """XFormSerializer""" url = serializers.HyperlinkedIdentityField( - view_name='xform-detail', lookup_field='pk') - owner = serializers.CharField(source='user.username') - project_name = serializers.CharField(source='project.name') + view_name="xform-detail", lookup_field="pk" + ) + owner = serializers.CharField(source="user.username") + project_name = serializers.CharField(source="project.name") class Meta: model = XForm - fields = ('url', 'id', 'id_string', 'title', 'num_of_submissions', - 'owner', 'project_id', 'project_name') + fields = ( + "url", + "id", + "id_string", + "title", + "num_of_submissions", + "owner", + "project_id", + "project_name", + ) class XFormListField(serializers.ManyRelatedField): @@ -140,33 +153,46 @@ class XFormListField(serializers.ManyRelatedField): def to_representation(self, iterable): return [ - dict(i) for i in XFormSerializer( - iterable, many=True, context=self.context).data + dict(i) + for i in XFormSerializer(iterable, many=True, context=self.context).data ] class MergedXFormSerializer(serializers.HyperlinkedModelSerializer): """MergedXForm Serializer to create and update merged datasets""" + url = serializers.HyperlinkedIdentityField( - view_name='merged-xform-detail', lookup_field='pk') - name = serializers.CharField( - max_length=XFORM_TITLE_LENGTH, write_only=True) + view_name="merged-xform-detail", lookup_field="pk" + ) + name = serializers.CharField(max_length=XFORM_TITLE_LENGTH, write_only=True) xforms = XFormListField( allow_empty=False, child_relation=serializers.HyperlinkedRelatedField( allow_empty=False, queryset=XForm.objects.filter( - is_merged_dataset=False, deleted_at__isnull=True), - view_name='xform-detail'), - validators=[minimum_two_xforms, has_matching_fields]) + is_merged_dataset=False, deleted_at__isnull=True + ), + view_name="xform-detail", + ), + validators=[minimum_two_xforms, has_matching_fields], + ) num_of_submissions = serializers.SerializerMethodField() last_submission_time = serializers.SerializerMethodField() class Meta: model = MergedXForm - fields = ('url', 'id', 'xforms', 'name', 'project', 'title', - 'num_of_submissions', 'last_submission_time', 'uuid') - write_only_fields = ('uuid', ) + fields = ( + "url", + "id", + "xforms", + "name", + "project", + "title", + "num_of_submissions", + "last_submission_time", + "uuid", + ) + write_only_fields = ("uuid",) # pylint: disable=no-self-use def get_num_of_submissions(self, obj): @@ -175,7 +201,7 @@ def get_num_of_submissions(self, obj): 'num_of_submissions'. """ - value = getattr(obj, 'number_of_submissions', obj.num_of_submissions) + value = getattr(obj, "number_of_submissions", obj.num_of_submissions) return value @@ -183,41 +209,41 @@ def get_last_submission_time(self, obj): """Return datetime of last submission from all forms""" values = [ x.last_submission_time.isoformat() - for x in obj.xforms.only('last_submission_time') + for x in obj.xforms.only("last_submission_time") if x.last_submission_time ] if values: return sorted(values, reverse=True)[0] def create(self, validated_data): - request = self.context['request'] - xforms = validated_data['xforms'] + request = self.context["request"] + xforms = validated_data["xforms"] # create merged xml, json with non conflicting id_string survey = get_merged_xform_survey(xforms) - survey['id_string'] = base64.b64encode( - uuid.uuid4().hex[:6].encode('utf-8')).decode('utf-8') - survey['sms_keyword'] = survey['id_string'] - survey['title'] = validated_data.pop('name') - validated_data['json'] = survey.to_json() + survey["id_string"] = base64.b64encode( + uuid.uuid4().hex[:6].encode("utf-8") + ).decode("utf-8") + survey["sms_keyword"] = survey["id_string"] + survey["title"] = validated_data.pop("name") + validated_data["json"] = survey.to_json() try: - validated_data['xml'] = survey.to_xml() + validated_data["xml"] = survey.to_xml() except PyXFormError as error: - raise serializers.ValidationError({ - 'xforms': - _("Problem Merging the Form: {}".format(error)) - }) - validated_data['user'] = validated_data['project'].user - validated_data['created_by'] = request.user - validated_data['is_merged_dataset'] = True - validated_data['num_of_submissions'] = sum( - [__.num_of_submissions for __ in validated_data.get('xforms')]) - validated_data['instances_with_geopoints'] = any([ - __.instances_with_geopoints for __ in validated_data.get('xforms') - ]) + raise serializers.ValidationError( + {"xforms": _("Problem Merging the Form: {}".format(error))} + ) + validated_data["user"] = validated_data["project"].user + validated_data["created_by"] = request.user + validated_data["is_merged_dataset"] = True + validated_data["num_of_submissions"] = sum( + [__.num_of_submissions for __ in validated_data.get("xforms")] + ) + validated_data["instances_with_geopoints"] = any( + [__.instances_with_geopoints for __ in validated_data.get("xforms")] + ) with transaction.atomic(): - instance = super(MergedXFormSerializer, - self).create(validated_data) + instance = super(MergedXFormSerializer, self).create(validated_data) if instance.xforms.all().count() == 0 and xforms: for xform in xforms: diff --git a/onadata/libs/serializers/metadata_serializer.py b/onadata/libs/serializers/metadata_serializer.py index 5444c0b552..bf92e52a1a 100644 --- a/onadata/libs/serializers/metadata_serializer.py +++ b/onadata/libs/serializers/metadata_serializer.py @@ -12,8 +12,8 @@ from django.core.validators import URLValidator from django.db.utils import IntegrityError from django.shortcuts import get_object_or_404 -from django.utils.translation import ugettext as _ -from future.moves.urllib.parse import urlparse +from django.utils.translation import gettext as _ +from six.moves.urllib.parse import urlparse from rest_framework import serializers from rest_framework.reverse import reverse @@ -21,42 +21,45 @@ from onadata.apps.logger.models import DataView, Instance, Project, XForm from onadata.apps.main.models import MetaData from onadata.libs.permissions import ROLES, ManagerRole -from onadata.libs.serializers.fields.instance_related_field import \ - InstanceRelatedField -from onadata.libs.serializers.fields.project_related_field import \ - ProjectRelatedField -from onadata.libs.serializers.fields.xform_related_field import \ - XFormRelatedField +from onadata.libs.serializers.fields.instance_related_field import InstanceRelatedField +from onadata.libs.serializers.fields.project_related_field import ProjectRelatedField +from onadata.libs.serializers.fields.xform_related_field import XFormRelatedField from onadata.libs.utils.common_tags import ( - XFORM_META_PERMS, SUBMISSION_REVIEW, IMPORTED_VIA_CSV_BY) + XFORM_META_PERMS, + SUBMISSION_REVIEW, + IMPORTED_VIA_CSV_BY, +) -UNIQUE_TOGETHER_ERROR = u"Object already exists" +UNIQUE_TOGETHER_ERROR = "Object already exists" -CSV_CONTENT_TYPE = 'text/csv' -MEDIA_TYPE = 'media' -DOC_TYPE = 'supporting_doc' +CSV_CONTENT_TYPE = "text/csv" +MEDIA_TYPE = "media" +DOC_TYPE = "supporting_doc" METADATA_TYPES = ( - ('data_license', _(u"Data License")), - ('enketo_preview_url', _(u"Enketo Preview URL")), - ('enketo_url', _(u"Enketo URL")), - ('form_license', _(u"Form License")), - ('mapbox_layer', _(u"Mapbox Layer")), - (MEDIA_TYPE, _(u"Media")), - ('public_link', _(u"Public Link")), - ('source', _(u"Source")), - (DOC_TYPE, _(u"Supporting Document")), - ('external_export', _(u"External Export")), - ('textit', _(u"TextIt")), - ('google_sheets', _(u"Google Sheet")), - ('xform_meta_perms', _("Xform meta permissions")), - ('submission_review', _("Submission Review")), - (IMPORTED_VIA_CSV_BY, _("Imported via CSV by"))) # yapf:disable - -DATAVIEW_TAG = 'dataview' -XFORM_TAG = 'xform' - -PROJECT_METADATA_TYPES = ((MEDIA_TYPE, _(u"Media")), - ('supporting_doc', _(u"Supporting Document"))) + ("data_license", _("Data License")), + ("enketo_preview_url", _("Enketo Preview URL")), + ("enketo_url", _("Enketo URL")), + ("form_license", _("Form License")), + ("mapbox_layer", _("Mapbox Layer")), + (MEDIA_TYPE, _("Media")), + ("public_link", _("Public Link")), + ("source", _("Source")), + (DOC_TYPE, _("Supporting Document")), + ("external_export", _("External Export")), + ("textit", _("TextIt")), + ("google_sheets", _("Google Sheet")), + ("xform_meta_perms", _("Xform meta permissions")), + ("submission_review", _("Submission Review")), + (IMPORTED_VIA_CSV_BY, _("Imported via CSV by")), +) # yapf:disable + +DATAVIEW_TAG = "dataview" +XFORM_TAG = "xform" + +PROJECT_METADATA_TYPES = ( + (MEDIA_TYPE, _("Media")), + ("supporting_doc", _("Supporting Document")), +) def get_linked_object(parts): @@ -72,34 +75,37 @@ def get_linked_object(parts): obj_pk = parts[1] try: obj_pk = int(obj_pk) - except ValueError: - raise serializers.ValidationError({ - 'data_value': - _(u"Invalid %(type)s id %(id)s." % - {'type': obj_type, - 'id': obj_pk}) - }) + except ValueError as e: + raise serializers.ValidationError( + { + "data_value": _( + "Invalid %(type)s id %(id)s." + % {"type": obj_type, "id": obj_pk} + ) + } + ) from e else: model = DataView if obj_type == DATAVIEW_TAG else XForm return get_object_or_404(model, pk=obj_pk) + return None class MetaDataSerializer(serializers.HyperlinkedModelSerializer): """ MetaData HyperlinkedModelSerializer """ - id = serializers.ReadOnlyField() # pylint: disable=C0103 + + id = serializers.ReadOnlyField() # pylint: disable=invalid-name xform = XFormRelatedField(queryset=XForm.objects.all(), required=False) - project = ProjectRelatedField( - queryset=Project.objects.all(), required=False) - instance = InstanceRelatedField( - queryset=Instance.objects.all(), required=False) + project = ProjectRelatedField(queryset=Project.objects.all(), required=False) + instance = InstanceRelatedField(queryset=Instance.objects.all(), required=False) data_value = serializers.CharField(max_length=255, required=True) data_type = serializers.ChoiceField(choices=METADATA_TYPES) data_file = serializers.FileField(required=False) data_file_type = serializers.CharField( - max_length=255, required=False, allow_blank=True) + max_length=255, required=False, allow_blank=True + ) media_url = serializers.SerializerMethodField() date_created = serializers.ReadOnlyField() @@ -108,106 +114,136 @@ class MetaDataSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = MetaData - fields = ('id', 'xform', 'project', 'instance', 'data_value', - 'data_type', 'data_file', 'data_file_type', 'media_url', - 'file_hash', 'url', 'date_created') + fields = ( + "id", + "xform", + "project", + "instance", + "data_value", + "data_type", + "data_file", + "data_file_type", + "media_url", + "file_hash", + "url", + "date_created", + ) def get_media_url(self, obj): """ Returns media URL for given metadata """ - if obj.data_type in [DOC_TYPE, MEDIA_TYPE] and\ - getattr(obj, "data_file") and getattr(obj.data_file, "url"): + if ( + obj.data_type in [DOC_TYPE, MEDIA_TYPE] + and getattr(obj, "data_file") + and getattr(obj.data_file, "url") + ): return obj.data_file.url - elif obj.data_type in [MEDIA_TYPE] and obj.is_linked_dataset: + if obj.data_type in [MEDIA_TYPE] and obj.is_linked_dataset: kwargs = { - 'kwargs': { - 'pk': obj.content_object.pk, - 'username': obj.content_object.user.username, - 'metadata': obj.pk + "kwargs": { + "pk": obj.content_object.pk, + "username": obj.content_object.user.username, + "metadata": obj.pk, }, - 'request': self.context.get('request'), - 'format': 'csv' + "request": self.context.get("request"), + "format": "csv", } - return reverse('xform-media', **kwargs) + return reverse("xform-media", **kwargs) + return None + # pylint: disable=too-many-branches def validate(self, attrs): """ Validate url if we are adding a media uri instead of a media file """ - value = attrs.get('data_value') - data_type = attrs.get('data_type') - data_file = attrs.get('data_file') - - if not ('project' in attrs or 'xform' in attrs or 'instance' in attrs): - raise serializers.ValidationError({ - 'missing_field': - _(u"`xform` or `project` or `instance`" - "field is required.") - }) + value = attrs.get("data_value") + data_type = attrs.get("data_type") + data_file = attrs.get("data_file") + + if not ("project" in attrs or "xform" in attrs or "instance" in attrs): + raise serializers.ValidationError( + { + "missing_field": _( + "`xform` or `project` or `instance`" "field is required." + ) + } + ) if data_file: allowed_types = settings.SUPPORTED_MEDIA_UPLOAD_TYPES - data_content_type = data_file.content_type \ - if data_file.content_type in allowed_types else \ - mimetypes.guess_type(data_file.name)[0] + # add geojson mimetype + mimetypes.add_type('application/geo+json', '.geojson') + data_content_type = ( + data_file.content_type + if data_file.content_type in allowed_types + else mimetypes.guess_type(data_file.name)[0] + ) if data_content_type not in allowed_types: - raise serializers.ValidationError({ - 'data_file': - _('Unsupported media file type %s' % data_content_type)}) - else: - attrs['data_file_type'] = data_content_type + raise serializers.ValidationError( + {"data_file": _(f"Unsupported media file type {data_content_type}")} + ) + attrs["data_file_type"] = data_content_type - if data_type == 'media' and data_file is None: + if data_type == "media" and data_file is None: try: URLValidator()(value) - except ValidationError: + except ValidationError as e: parts = value.split() if len(parts) < 3: - raise serializers.ValidationError({ - 'data_value': - _(u"Expecting 'xform [xform id] [media name]' " - "or 'dataview [dataview id] [media name]' " - "or a valid URL.") - }) + raise serializers.ValidationError( + { + "data_value": _( + "Expecting 'xform [xform id] [media name]' " + "or 'dataview [dataview id] [media name]' " + "or a valid URL." + ) + } + ) obj = get_linked_object(parts) if obj: xform = obj.xform if isinstance(obj, DataView) else obj - request = self.context['request'] + request = self.context["request"] user_has_role = ManagerRole.user_has_role - has_perm = user_has_role(request.user, xform) or \ - user_has_role(request.user, obj.project) + has_perm = user_has_role(request.user, xform) or user_has_role( + request.user, obj.project + ) if not has_perm: - raise serializers.ValidationError({ - 'data_value': - _(u"User has no permission to " - "the dataview.") - }) + raise serializers.ValidationError( + { + "data_value": _( + "User has no permission to " "the dataview." + ) + } + ) from e else: - raise serializers.ValidationError({ - 'data_value': - _(u"Invalid url '%s'." % value) - }) + raise serializers.ValidationError( + {"data_value": _(f"Invalid url '{value}'.")} + ) from e else: # check if we have a value for the filename. if not os.path.basename(urlparse(value).path): - raise serializers.ValidationError({ - 'data_value': - _(u"Cannot get filename from URL %s. URL should " - u"include the filename e.g " - u"http://example.com/data.csv" % value) - }) + raise serializers.ValidationError( + { + "data_value": _( + f"Cannot get filename from URL {value}. URL should " + "include the filename e.g " + "http://example.com/data.csv" + ) + } + ) if data_type == XFORM_META_PERMS: - perms = value.split('|') + perms = value.split("|") if len(perms) != 2 or not set(perms).issubset(set(ROLES)): raise serializers.ValidationError( - _(u"Format 'role'|'role' or Invalid role")) + _("Format 'role'|'role' or Invalid role") + ) return attrs - # pylint: disable=R0201 + # pylint: disable=no-self-use def get_content_object(self, validated_data): """ Returns the validated 'xform' or 'project' or 'instance' ids being @@ -215,46 +251,52 @@ def get_content_object(self, validated_data): """ if validated_data: - return (validated_data.get('xform') or - validated_data.get('project') or - validated_data.get('instance')) + return ( + validated_data.get("xform") + or validated_data.get("project") + or validated_data.get("instance") + ) + return None def create(self, validated_data): - data_type = validated_data.get('data_type') - data_file = validated_data.get('data_file') - data_file_type = validated_data.get('data_file_type') + data_type = validated_data.get("data_type") + data_file = validated_data.get("data_file") + data_file_type = validated_data.get("data_file_type") content_object = self.get_content_object(validated_data) - data_value = data_file.name \ - if data_file else validated_data.get('data_value') + data_value = data_file.name if data_file else validated_data.get("data_value") # not exactly sure what changed in the requests.FILES for django 1.7 # csv files uploaded in windows do not have the text/csv content_type # this works around that - if data_type == MEDIA_TYPE and data_file \ - and data_file.name.lower().endswith('.csv') \ - and data_file_type != CSV_CONTENT_TYPE: + if ( + data_type == MEDIA_TYPE + and data_file + and data_file.name.lower().endswith(".csv") + and data_file_type != CSV_CONTENT_TYPE + ): data_file_type = CSV_CONTENT_TYPE content_type = ContentType.objects.get_for_model(content_object) try: if data_type == XFORM_META_PERMS: - metadata = \ - MetaData.xform_meta_permission(content_object, - data_value=data_value) + metadata = MetaData.xform_meta_permission( + content_object, data_value=data_value + ) update_role_by_meta_xform_perms(content_object) elif data_type == SUBMISSION_REVIEW: # ensure only one submission_review metadata exists per form if MetaData.submission_review(content_object): raise serializers.ValidationError(_(UNIQUE_TOGETHER_ERROR)) - else: - metadata = MetaData.submission_review( - content_object, data_value=data_value) + metadata = MetaData.submission_review( + content_object, data_value=data_value + ) elif data_type == IMPORTED_VIA_CSV_BY: metadata = MetaData.instance_csv_imported_by( - content_object, data_value=data_value) + content_object, data_value=data_value + ) else: metadata = MetaData.objects.create( content_type=content_type, @@ -262,15 +304,16 @@ def create(self, validated_data): data_value=data_value, data_file=data_file, data_file_type=data_file_type, - object_id=content_object.id) + object_id=content_object.id, + ) return metadata - except IntegrityError: - raise serializers.ValidationError(_(UNIQUE_TOGETHER_ERROR)) + except IntegrityError as e: + raise serializers.ValidationError(_(UNIQUE_TOGETHER_ERROR)) from e + return None def update(self, instance, validated_data): - instance = super(MetaDataSerializer, self).update( - instance, validated_data) + instance = super().update(instance, validated_data) if instance.data_type == XFORM_META_PERMS: update_role_by_meta_xform_perms(instance.content_object) diff --git a/onadata/libs/serializers/note_serializer.py b/onadata/libs/serializers/note_serializer.py index 56a155ea86..8eb77c9003 100644 --- a/onadata/libs/serializers/note_serializer.py +++ b/onadata/libs/serializers/note_serializer.py @@ -2,7 +2,7 @@ """ Note Serializers Module """ -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from guardian.shortcuts import assign_perm from rest_framework import exceptions, serializers diff --git a/onadata/libs/serializers/organization_member_serializer.py b/onadata/libs/serializers/organization_member_serializer.py index 4a0fb0baad..09df7131fe 100644 --- a/onadata/libs/serializers/organization_member_serializer.py +++ b/onadata/libs/serializers/organization_member_serializer.py @@ -1,5 +1,5 @@ from django.contrib.auth.models import User -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.core.mail import send_mail from rest_framework import serializers diff --git a/onadata/libs/serializers/organization_serializer.py b/onadata/libs/serializers/organization_serializer.py index f0c4c710ba..a000e611df 100644 --- a/onadata/libs/serializers/organization_serializer.py +++ b/onadata/libs/serializers/organization_serializer.py @@ -2,86 +2,92 @@ """ Organization Serializer """ -from past.builtins import basestring # pylint: disable=redefined-builtin -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.db.models.query import QuerySet -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import serializers from onadata.apps.api import tools from onadata.apps.api.models import OrganizationProfile -from onadata.apps.api.tools import (_get_first_last_names, - get_organization_members, - get_organization_owners) +from onadata.apps.api.tools import ( + _get_first_last_names, + get_organization_members, + get_organization_owners, +) from onadata.apps.main.forms import RegistrationFormUserProfile from onadata.libs.permissions import get_role_in_org from onadata.libs.serializers.fields.json_field import JsonField +# pylint: disable=invalid-name +User = get_user_model() + class OrganizationSerializer(serializers.HyperlinkedModelSerializer): """ Organization profile serializer """ + url = serializers.HyperlinkedIdentityField( - view_name='organizationprofile-detail', lookup_field='user') - org = serializers.CharField(source='user.username', max_length=30) + view_name="organizationprofile-detail", lookup_field="user" + ) + org = serializers.CharField(source="user.username", max_length=30) user = serializers.HyperlinkedRelatedField( - view_name='user-detail', lookup_field='username', read_only=True) + view_name="user-detail", lookup_field="username", read_only=True + ) creator = serializers.HyperlinkedRelatedField( - view_name='user-detail', lookup_field='username', read_only=True) + view_name="user-detail", lookup_field="username", read_only=True + ) users = serializers.SerializerMethodField() metadata = JsonField(required=False) name = serializers.CharField(max_length=30) class Meta: model = OrganizationProfile - exclude = ('created_by', 'is_organization', 'organization') - owner_only_fields = ('metadata', ) + exclude = ("created_by", "is_organization", "organization") + owner_only_fields = ("metadata",) def __init__(self, *args, **kwargs): - super(OrganizationSerializer, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) - if self.instance and hasattr(self.Meta, 'owner_only_fields'): - request = self.context.get('request') + if self.instance and hasattr(self.Meta, "owner_only_fields"): + request = self.context.get("request") is_permitted = ( - request and request.user and - request.user.has_perm('api.view_organizationprofile', - self.instance)) - if isinstance(self.instance, QuerySet) or not is_permitted or \ - not request: - for field in getattr(self.Meta, 'owner_only_fields'): + request + and request.user + and request.user.has_perm("api.view_organizationprofile", self.instance) + ) + if isinstance(self.instance, QuerySet) or not is_permitted or not request: + for field in getattr(self.Meta, "owner_only_fields"): self.fields.pop(field) def update(self, instance, validated_data): + """Update organization profile properties.""" # update the user model - if 'name' in validated_data: - first_name, last_name = \ - _get_first_last_names(validated_data.get('name')) + if "name" in validated_data: + first_name, last_name = _get_first_last_names(validated_data.get("name")) instance.user.first_name = first_name instance.user.last_name = last_name instance.user.save() - return super(OrganizationSerializer, self).update( - instance, validated_data - ) + return super().update(instance, validated_data) def create(self, validated_data): - org = validated_data.get('user') + """Create an organization profile.""" + org = validated_data.get("user") if org: - org = org.get('username') + org = org.get("username") - org_name = validated_data.get('name', None) + org_name = validated_data.get("name", None) creator = None - if 'request' in self.context: - creator = self.context['request'].user + if "request" in self.context: + creator = self.context["request"].user - validated_data['organization'] = org_name + validated_data["organization"] = org_name - profile = tools.create_organization_object(org, creator, - validated_data) + profile = tools.create_organization_object(org, creator, validated_data) profile.save() return profile @@ -90,47 +96,50 @@ def validate_org(self, value): # pylint: disable=no-self-use """ Validate organization name. """ - org = value.lower() if isinstance(value, basestring) else value + org = value.lower() if isinstance(value, str) else value if org in RegistrationFormUserProfile.RESERVED_USERNAMES: - raise serializers.ValidationError(_( - u"%s is a reserved name, please choose another" % org - )) - elif not RegistrationFormUserProfile.legal_usernames_re.search(org): - raise serializers.ValidationError(_( - u"Organization may only contain alpha-numeric characters and " - u"underscores" - )) + raise serializers.ValidationError( + _(f"{org} is a reserved name, please choose another") + ) + if not RegistrationFormUserProfile.legal_usernames_re.search(org): + raise serializers.ValidationError( + _( + "Organization may only contain alpha-numeric characters and " + "underscores" + ) + ) try: User.objects.get(username=org) except User.DoesNotExist: return org - raise serializers.ValidationError(_( - u"Organization %s already exists." % org - )) + raise serializers.ValidationError(_(f"Organization {org} already exists.")) def get_users(self, obj): # pylint: disable=no-self-use """ Return organization members. """ - def create_user_list(user_list): - return [{ - 'user': u.username, - 'role': get_role_in_org(u, obj), - 'first_name': u.first_name, - 'last_name': u.last_name, - 'gravatar': u.profile.gravatar - } for u in user_list] + + def _create_user_list(user_list): + return [ + { + "user": u.username, + "role": get_role_in_org(u, obj), + "first_name": u.first_name, + "last_name": u.last_name, + "gravatar": u.profile.gravatar, + } + for u in user_list + ] members = get_organization_members(obj) if obj else [] owners = get_organization_owners(obj) if obj else [] if owners and members: - members = members.exclude( - username__in=[user.username for user in owners]) + members = members.exclude(username__in=[user.username for user in owners]) - members_list = create_user_list(members) - owners_list = create_user_list(owners) + members_list = _create_user_list(members) + owners_list = _create_user_list(owners) return owners_list + members_list diff --git a/onadata/libs/serializers/password_reset_serializer.py b/onadata/libs/serializers/password_reset_serializer.py index ce292cf95e..01ab28888c 100644 --- a/onadata/libs/serializers/password_reset_serializer.py +++ b/onadata/libs/serializers/password_reset_serializer.py @@ -1,83 +1,117 @@ -from builtins import bytes as b -from future.moves.urllib.parse import urlparse +# -*- coding: utf-8 -*- +""" +Password reset serializer. +""" +from six.moves.urllib.parse import urlparse -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.contrib.auth.tokens import PasswordResetTokenGenerator from django.core.mail import send_mail from django.template import loader from django.utils.encoding import force_bytes from django.utils.http import urlsafe_base64_decode from django.utils.http import urlsafe_base64_encode -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from onadata.libs.utils.user_auth import invalidate_and_regen_tokens +# pylint: disable=invalid-name +User = get_user_model() + + class CustomPasswordResetTokenGenerator(PasswordResetTokenGenerator): - """Custom Password Token Generator Class.""" + """ + Custom Password Token Generator Class. + """ + def _make_hash_value(self, user, timestamp): # Include user email alongside user password to the generated token # as the user state object that might change after a password reset # to produce a token that invalidated. - login_timestamp = '' if user.last_login is None\ + login_timestamp = ( + "" + if user.last_login is None else user.last_login.replace(microsecond=0, tzinfo=None) - return str(user.pk) + user.password + user.email +\ - str(login_timestamp) + str(timestamp) + ) + return ( + str(user.pk) + + user.password + + user.email + + str(login_timestamp) + + str(timestamp) + ) default_token_generator = CustomPasswordResetTokenGenerator() -def get_password_reset_email(user, reset_url, - subject_template_name='registration/password_reset_subject.txt', # noqa - email_template_name='api_password_reset_email.html', # noqa - token_generator=default_token_generator, - email_subject=None): +# pylint: disable=unused-argument,too-many-arguments +def get_password_reset_email( + user, + reset_url, + subject_template_name="registration/password_reset_subject.txt", # noqa + email_template_name="api_password_reset_email.html", # noqa + token_generator=default_token_generator, + email_subject=None, +): """Creates the subject and email body for password reset email.""" result = urlparse(reset_url) site_name = domain = result.hostname - encoded_username = urlsafe_base64_encode(b(user.username.encode('utf-8'))) + encoded_username = urlsafe_base64_encode(bytes(user.username.encode("utf-8"))) c = { - 'email': user.email, - 'domain': domain, - 'path': result.path, - 'site_name': site_name, - 'uid': urlsafe_base64_encode(force_bytes(user.pk)), - 'username': user.username, - 'encoded_username': encoded_username, - 'token': token_generator.make_token(user), - 'protocol': result.scheme if result.scheme != '' else 'http', + "email": user.email, + "domain": domain, + "path": result.path, + "site_name": site_name, + "uid": urlsafe_base64_encode(force_bytes(user.pk)), + "username": user.username, + "encoded_username": encoded_username, + "token": token_generator.make_token(user), + "protocol": result.scheme if result.scheme != "" else "http", } # if subject email provided don't load the subject template - subject = email_subject or loader.render_to_string(subject_template_name, - c) + subject = email_subject or loader.render_to_string(subject_template_name, c) # Email subject *must not* contain newlines - subject = ''.join(subject.splitlines()) + subject = "".join(subject.splitlines()) email = loader.render_to_string(email_template_name, c) return subject, email def get_user_from_uid(uid): + """ + Return user from base64 encoded ``uid``. + """ if uid is None: raise serializers.ValidationError(_("uid is required!")) try: uid = urlsafe_base64_decode(uid) user = User.objects.get(pk=uid) - except (TypeError, ValueError, OverflowError, User.DoesNotExist): - raise serializers.ValidationError(_(u"Invalid uid %s") % uid) + except (TypeError, ValueError, OverflowError, User.DoesNotExist) as e: + raise serializers.ValidationError(_("Invalid uid {uid}")) from e return user -class PasswordResetChange(object): +# pylint: disable=too-few-public-methods +class PasswordResetChange: + """ + Class resets and changes the password. + + Class imitates a model functionality for use with PasswordResetSerializer + """ + def __init__(self, uid, new_password, token): self.uid = uid self.new_password = new_password self.token = token def save(self): + """ + Set a new user password and invalidate/regenerate tokens. + """ user = get_user_from_uid(self.uid) if user: user.set_password(self.new_password) @@ -85,17 +119,27 @@ def save(self): invalidate_and_regen_tokens(user) -class PasswordReset(object): +# pylint: disable=too-few-public-methods +class PasswordReset: + """ + Class resets the password and sends the reset email. + + Class imitates a model functionality for use with PasswordResetSerializer + """ + def __init__(self, email, reset_url, email_subject=None): self.email = email self.reset_url = reset_url self.email_subject = email_subject - def save(self, - subject_template_name='registration/password_reset_subject.txt', - email_template_name='api_password_reset_email.html', - token_generator=default_token_generator, - from_email=None): + # pylint: disable=unused-argument + def save( + self, + subject_template_name="registration/password_reset_subject.txt", + email_template_name="api_password_reset_email.html", + token_generator=default_token_generator, + from_email=None, + ): """ Generates a one-use only link for resetting password and sends to the user. @@ -110,36 +154,51 @@ def save(self, if not user.has_usable_password(): continue subject, email = get_password_reset_email( - user, reset_url, subject_template_name, email_template_name, - email_subject=self.email_subject) + user, + reset_url, + subject_template_name, + email_template_name, + email_subject=self.email_subject, + ) send_mail(subject, email, from_email, [user.email]) +# pylint: disable=abstract-method class PasswordResetSerializer(serializers.Serializer): + """ + Password reset serializer. + """ + email = serializers.EmailField(label=_("Email"), max_length=254) reset_url = serializers.URLField(label=_("Reset URL"), max_length=254) - email_subject = serializers.CharField(label=_("Email Subject"), - required=False, max_length=78, - allow_blank=True) + email_subject = serializers.CharField( + label=_("Email Subject"), required=False, max_length=78, allow_blank=True + ) + # pylint: disable=no-self-use def validate_email(self, value): + """ + Validates the email. + """ users = User.objects.filter(email__iexact=value) if users.count() == 0: - raise serializers.ValidationError(_( - u"User '%(value)s' does not exist." % {"value": value} - )) + raise serializers.ValidationError(_(f"User '{value}' does not exist.")) return value def validate_email_subject(self, value): - if len(value) == 0: + """ + Validate the email subject is not empty. + """ + if value: return None return value def create(self, validated_data): + """Reset a user password.""" instance = PasswordReset(**validated_data) instance.save() @@ -147,25 +206,35 @@ def create(self, validated_data): class PasswordResetChangeSerializer(serializers.Serializer): + """ + Reset and change password serializer. + """ + uid = serializers.CharField(max_length=50) new_password = serializers.CharField(min_length=4, max_length=128) token = serializers.CharField(max_length=128) + # pylint: disable=no-self-use def validate_uid(self, value): + """ + Validate the user uid. + """ get_user_from_uid(value) return value def validate(self, attrs): - user = get_user_from_uid(attrs.get('uid')) - token = attrs.get('token') + """Validates the generated user token.""" + user = get_user_from_uid(attrs.get("uid")) + token = attrs.get("token") if not default_token_generator.check_token(user, token): - raise serializers.ValidationError(_("Invalid token: %s") % token) + raise serializers.ValidationError(_(f"Invalid token: {token}")) return attrs def create(self, validated_data, instance=None): + """Set a new user password and invalidate/regenerate tokens.""" instance = PasswordResetChange(**validated_data) instance.save() diff --git a/onadata/libs/serializers/project_serializer.py b/onadata/libs/serializers/project_serializer.py index 1d00d673e9..24c611ac3e 100644 --- a/onadata/libs/serializers/project_serializer.py +++ b/onadata/libs/serializers/project_serializer.py @@ -2,41 +2,60 @@ """ Project Serializer module. """ -from future.utils import listvalues +from six import itervalues from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.cache import cache from django.db.utils import IntegrityError -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import serializers from onadata.apps.api.models import OrganizationProfile -from onadata.apps.api.tools import (get_organization_members_team, - get_or_create_organization_owners_team) +from onadata.apps.api.tools import ( + get_or_create_organization_owners_team, + get_organization_members_team, +) from onadata.apps.logger.models import Project, XForm from onadata.libs.permissions import ( - OwnerRole, ReadOnlyRole, ManagerRole, get_role, is_organization) -from onadata.libs.serializers.dataview_serializer import \ - DataViewMinimalSerializer + ManagerRole, + OwnerRole, + ReadOnlyRole, + get_role, + is_organization, +) +from onadata.libs.serializers.dataview_serializer import DataViewMinimalSerializer from onadata.libs.serializers.fields.json_field import JsonField from onadata.libs.serializers.tag_list_serializer import TagListSerializer from onadata.libs.utils.analytics import track_object_event from onadata.libs.utils.cache_tools import ( - PROJ_BASE_FORMS_CACHE, PROJ_FORMS_CACHE, PROJ_NUM_DATASET_CACHE, - PROJ_PERM_CACHE, PROJ_SUB_DATE_CACHE, PROJ_TEAM_USERS_CACHE, - PROJECT_LINKED_DATAVIEWS, PROJ_OWNER_CACHE, safe_delete) + PROJ_BASE_FORMS_CACHE, + PROJ_FORMS_CACHE, + PROJ_NUM_DATASET_CACHE, + PROJ_OWNER_CACHE, + PROJ_PERM_CACHE, + PROJ_SUB_DATE_CACHE, + PROJ_TEAM_USERS_CACHE, + PROJECT_LINKED_DATAVIEWS, + safe_delete, +) from onadata.libs.utils.decorators import check_obj +# pylint: disable=invalid-name +User = get_user_model() + def get_project_xforms(project): """ Returns an XForm queryset from project. The prefetched `xforms_prefetch` or `xform_set.filter()` queryset. """ - return (project.xforms_prefetch if hasattr(project, 'xforms_prefetch') else - project.xform_set.filter(deleted_at__isnull=True)) + return ( + project.xforms_prefetch + if hasattr(project, "xforms_prefetch") + else project.xform_set.filter(deleted_at__isnull=True) + ) @check_obj @@ -46,20 +65,18 @@ def get_last_submission_date(project): :param project: The project to find the last submission date for. """ - last_submission_date = cache.get( - '{}{}'.format(PROJ_SUB_DATE_CACHE, project.pk)) + cache_key = f"{PROJ_SUB_DATE_CACHE}{project.pk}" + last_submission_date = cache.get(cache_key) if last_submission_date: return last_submission_date xforms = get_project_xforms(project) dates = [ - x.last_submission_time for x in xforms - if x.last_submission_time is not None + x.last_submission_time for x in xforms if x.last_submission_time is not None ] dates.sort(reverse=True) last_submission_date = dates[0] if dates else None - cache.set('{}{}'.format(PROJ_SUB_DATE_CACHE, project.pk), - last_submission_date) + cache.set(cache_key, last_submission_date) return last_submission_date @@ -70,12 +87,13 @@ def get_num_datasets(project): :param project: The project to find datasets for. """ - count = cache.get('{}{}'.format(PROJ_NUM_DATASET_CACHE, project.pk)) + project_cache_key = f"{PROJ_NUM_DATASET_CACHE}{project.pk}" + count = cache.get(project_cache_key) if count: return count count = len(get_project_xforms(project)) - cache.set('{}{}'.format(PROJ_NUM_DATASET_CACHE, project.pk), count) + cache.set(project_cache_key, count) return count @@ -91,8 +109,8 @@ def get_team_permissions(team, project): Return team permissions. """ return project.projectgroupobjectpermission_set.filter( - group__pk=team.pk).values_list( - 'permission__codename', flat=True) + group__pk=team.pk + ).values_list("permission__codename", flat=True) @check_obj @@ -100,7 +118,8 @@ def get_teams(project): """ Return the teams with access to the project. """ - teams_users = cache.get('{}{}'.format(PROJ_TEAM_USERS_CACHE, project.pk)) + project_team_cache_key = f"{PROJ_TEAM_USERS_CACHE}{project.pk}" + teams_users = cache.get(project_team_cache_key) if teams_users: return teams_users @@ -112,13 +131,11 @@ def get_teams(project): users = [user.username for user in team.user_set.all()] perms = get_team_permissions(team, project) - teams_users.append({ - "name": team.name, - "role": get_role(perms, project), - "users": users - }) + teams_users.append( + {"name": team.name, "role": get_role(perms, project), "users": users} + ) - cache.set('{}{}'.format(PROJ_TEAM_USERS_CACHE, project.pk), teams_users) + cache.set(project_team_cache_key, teams_users) return teams_users @@ -127,23 +144,25 @@ def get_users(project, context, all_perms=True): """ Return a list of users and organizations that have access to the project. """ + project_permissions_cache_key = f"{PROJ_PERM_CACHE}{project.pk}" if all_perms: - users = cache.get('{}{}'.format(PROJ_PERM_CACHE, project.pk)) + users = cache.get(project_permissions_cache_key) if users: return users data = {} - request_user = context['request'].user + request_user = context["request"].user if not request_user.is_anonymous: request_user_perms = [ perm.permission.codename for perm in project.projectuserobjectpermission_set.filter( - user=request_user)] + user=request_user + ) + ] request_user_role = get_role(request_user_perms, project) - request_user_is_admin = request_user_role in [ - OwnerRole.name, ManagerRole.name] + request_user_is_admin = request_user_role in [OwnerRole.name, ManagerRole.name] else: request_user_is_admin = False @@ -151,29 +170,31 @@ def get_users(project, context, all_perms=True): if perm.user_id not in data: user = perm.user - if all_perms or user in [ - request_user, project.organization - ] or request_user_is_admin: + if ( + all_perms + or user in [request_user, project.organization] + or request_user_is_admin + ): data[perm.user_id] = { - 'permissions': [], - 'is_org': is_organization(user.profile), - 'metadata': user.profile.metadata, - 'first_name': user.first_name, - 'last_name': user.last_name, - 'user': user.username + "permissions": [], + "is_org": is_organization(user.profile), + "metadata": user.profile.metadata, + "first_name": user.first_name, + "last_name": user.last_name, + "user": user.username, } if perm.user_id in data: - data[perm.user_id]['permissions'].append(perm.permission.codename) + data[perm.user_id]["permissions"].append(perm.permission.codename) for k in list(data): - data[k]['permissions'].sort() - data[k]['role'] = get_role(data[k]['permissions'], project) - del data[k]['permissions'] + data[k]["permissions"].sort() + data[k]["role"] = get_role(data[k]["permissions"], project) + del data[k]["permissions"] - results = listvalues(data) + results = list(itervalues(data)) if all_perms: - cache.set('{}{}'.format(PROJ_PERM_CACHE, project.pk), results) + cache.set(project_permissions_cache_key, results) return results @@ -183,65 +204,85 @@ def set_owners_permission(user, project): OwnerRole.add(user, project) +# pylint: disable=too-few-public-methods class BaseProjectXFormSerializer(serializers.HyperlinkedModelSerializer): """ BaseProjectXFormSerializer class. """ - formid = serializers.ReadOnlyField(source='id') - name = serializers.ReadOnlyField(source='title') + formid = serializers.ReadOnlyField(source="id") + name = serializers.ReadOnlyField(source="title") + + # pylint: disable=too-few-public-methods,missing-class-docstring class Meta: model = XForm - fields = ('name', 'formid', 'id_string', 'is_merged_dataset') + fields = ("name", "formid", "id_string", "is_merged_dataset") +# pylint: disable=too-few-public-methods class ProjectXFormSerializer(serializers.HyperlinkedModelSerializer): """ ProjectXFormSerializer class - to return project xform info. """ + url = serializers.HyperlinkedIdentityField( - view_name='xform-detail', lookup_field='pk') - formid = serializers.ReadOnlyField(source='id') - name = serializers.ReadOnlyField(source='title') + view_name="xform-detail", lookup_field="pk" + ) + formid = serializers.ReadOnlyField(source="id") + name = serializers.ReadOnlyField(source="title") published_by_formbuilder = serializers.SerializerMethodField() + # pylint: disable=too-few-public-methods,missing-class-docstring class Meta: model = XForm - fields = ('name', 'formid', 'id_string', 'num_of_submissions', - 'downloadable', 'encrypted', 'published_by_formbuilder', - 'last_submission_time', 'date_created', 'url', - 'last_updated_at', 'is_merged_dataset', ) + fields = ( + "name", + "formid", + "id_string", + "num_of_submissions", + "downloadable", + "encrypted", + "published_by_formbuilder", + "last_submission_time", + "date_created", + "url", + "last_updated_at", + "is_merged_dataset", + ) def get_published_by_formbuilder(self, obj): # pylint: disable=no-self-use """ Returns true if the form was published by formbuilder. """ - metadata = obj.metadata_set.filter( - data_type='published_by_formbuilder').first() - return (metadata and hasattr(metadata, 'data_value') - and metadata.data_value) + metadata = obj.metadata_set.filter(data_type="published_by_formbuilder").first() + return metadata and hasattr(metadata, "data_value") and metadata.data_value class BaseProjectSerializer(serializers.HyperlinkedModelSerializer): """ BaseProjectSerializer class. """ - projectid = serializers.ReadOnlyField(source='id') + + projectid = serializers.ReadOnlyField(source="id") url = serializers.HyperlinkedIdentityField( - view_name='project-detail', lookup_field='pk') + view_name="project-detail", lookup_field="pk" + ) owner = serializers.HyperlinkedRelatedField( - view_name='user-detail', - source='organization', - lookup_field='username', + view_name="user-detail", + source="organization", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME)) + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), + ) created_by = serializers.HyperlinkedRelatedField( - view_name='user-detail', lookup_field='username', read_only=True) + view_name="user-detail", lookup_field="username", read_only=True + ) metadata = JsonField(required=False) starred = serializers.SerializerMethodField() users = serializers.SerializerMethodField() forms = serializers.SerializerMethodField() - public = serializers.BooleanField(source='shared') + public = serializers.BooleanField(source="shared") tags = TagListSerializer(read_only=True) num_datasets = serializers.SerializerMethodField() last_submission_date = serializers.SerializerMethodField() @@ -250,25 +291,39 @@ class BaseProjectSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Project fields = [ - 'url', 'projectid', 'owner', 'created_by', 'metadata', 'starred', - 'users', 'forms', 'public', 'tags', 'num_datasets', - 'last_submission_date', 'teams', 'name', 'date_created', - 'date_modified', 'deleted_at' + "url", + "projectid", + "owner", + "created_by", + "metadata", + "starred", + "users", + "forms", + "public", + "tags", + "num_datasets", + "last_submission_date", + "teams", + "name", + "date_created", + "date_modified", + "deleted_at", ] def get_starred(self, obj): """ Return True if request user has starred this project. """ - return is_starred(obj, self.context['request']) + return is_starred(obj, self.context["request"]) def get_users(self, obj): """ Return a list of users and organizations that have access to the project. """ - owner_query_param_in_request = 'request' in self.context and\ - "owner" in self.context['request'].GET + owner_query_param_in_request = ( + "request" in self.context and "owner" in self.context["request"].GET + ) return get_users(obj, self.context, owner_query_param_in_request) @check_obj @@ -276,16 +331,18 @@ def get_forms(self, obj): """ Return list of xforms in the project. """ - forms = cache.get('{}{}'.format(PROJ_BASE_FORMS_CACHE, obj.pk)) + project_forms_cache_key = f"{PROJ_BASE_FORMS_CACHE}{obj.pk}" + forms = cache.get(project_forms_cache_key) if forms: return forms xforms = get_project_xforms(obj) - request = self.context.get('request') + request = self.context.get("request") serializer = BaseProjectXFormSerializer( - xforms, context={'request': request}, many=True) + xforms, context={"request": request}, many=True + ) forms = list(serializer.data) - cache.set('{}{}'.format(PROJ_BASE_FORMS_CACHE, obj.pk), forms) + cache.set(project_forms_cache_key, forms) return forms @@ -312,11 +369,12 @@ def can_add_project_to_profile(user, organization): """ Check if user has permission to add a project to a profile. """ - perms = 'can_add_project' - if user != organization and \ - not user.has_perm(perms, organization.profile) and \ - not user.has_perm( - perms, OrganizationProfile.objects.get(user=organization)): + perms = "can_add_project" + if ( + user != organization + and not user.has_perm(perms, organization.profile) + and not user.has_perm(perms, OrganizationProfile.objects.get(user=organization)) + ): return False return True @@ -326,22 +384,27 @@ class ProjectSerializer(serializers.HyperlinkedModelSerializer): """ ProjectSerializer class - creates and updates a project. """ - projectid = serializers.ReadOnlyField(source='id') + + projectid = serializers.ReadOnlyField(source="id") url = serializers.HyperlinkedIdentityField( - view_name='project-detail', lookup_field='pk') + view_name="project-detail", lookup_field="pk" + ) owner = serializers.HyperlinkedRelatedField( - view_name='user-detail', - source='organization', - lookup_field='username', + view_name="user-detail", + source="organization", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME)) + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), + ) created_by = serializers.HyperlinkedRelatedField( - view_name='user-detail', lookup_field='username', read_only=True) + view_name="user-detail", lookup_field="username", read_only=True + ) metadata = JsonField(required=False) starred = serializers.SerializerMethodField() users = serializers.SerializerMethodField() forms = serializers.SerializerMethodField() - public = serializers.BooleanField(source='shared') + public = serializers.BooleanField(source="shared") tags = TagListSerializer(read_only=True) num_datasets = serializers.SerializerMethodField() last_submission_date = serializers.SerializerMethodField() @@ -350,21 +413,24 @@ class ProjectSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Project - exclude = ('shared', 'user_stars', 'deleted_by', 'organization') + exclude = ("shared", "user_stars", "deleted_by", "organization") def validate(self, attrs): - name = attrs.get('name') - organization = attrs.get('organization') + """Validate the project name does not exist and the user has the permissions to + create a project in the organization.""" + name = attrs.get("name") + organization = attrs.get("organization") if not self.instance and organization: project_w_same_name = Project.objects.filter( - name__iexact=name, - organization=organization) + name__iexact=name, organization=organization + ) if project_w_same_name: - raise serializers.ValidationError({ - 'name': _(u"Project {} already exists.".format(name))}) + raise serializers.ValidationError( + {"name": _(f"Project {name} already exists.")} + ) else: organization = organization or self.instance.organization - request = self.context['request'] + request = self.context["request"] try: has_perm = can_add_project_to_profile(request.user, organization) except OrganizationProfile.DoesNotExist: @@ -372,12 +438,14 @@ def validate(self, attrs): # A user does not require permissions to the user's account forms. has_perm = False if not has_perm: - raise serializers.ValidationError({ - 'owner': - _("You do not have permission to create a project " - "in the organization %(organization)s." % { - 'organization': organization}) - }) + raise serializers.ValidationError( + { + "owner": _( + "You do not have permission to create a project " + f"in the organization {organization}." + ) + } + ) return attrs def validate_public(self, value): # pylint: disable=no-self-use @@ -386,7 +454,8 @@ def validate_public(self, value): # pylint: disable=no-self-use """ if not settings.ALLOW_PUBLIC_DATASETS and value: raise serializers.ValidationError( - _('Public projects are currently disabled.')) + _("Public projects are currently disabled.") + ) return value def validate_metadata(self, value): # pylint: disable=no-self-use @@ -396,90 +465,96 @@ def validate_metadata(self, value): # pylint: disable=no-self-use msg = serializers.ValidationError(_("Invaid value for metadata")) try: json_val = JsonField.to_json(value) - except ValueError: - raise serializers.ValidationError(msg) + except ValueError as e: + raise serializers.ValidationError(msg) from e else: if json_val is None: raise serializers.ValidationError(msg) return value def update(self, instance, validated_data): - metadata = JsonField.to_json(validated_data.get('metadata')) + """Update project properties.""" + metadata = JsonField.to_json(validated_data.get("metadata")) if metadata is None: - metadata = dict() - owner = validated_data.get('organization') + metadata = {} + owner = validated_data.get("organization") if self.partial and metadata: if not isinstance(instance.metadata, dict): instance.metadata = {} instance.metadata.update(metadata) - validated_data['metadata'] = instance.metadata + validated_data["metadata"] = instance.metadata if self.partial and owner: # give the new owner permissions set_owners_permission(owner, instance) if is_organization(owner.profile): - owners_team = get_or_create_organization_owners_team( - owner.profile) + owners_team = get_or_create_organization_owners_team(owner.profile) members_team = get_organization_members_team(owner.profile) OwnerRole.add(owners_team, instance) ReadOnlyRole.add(members_team, instance) owners = owners_team.user_set.all() # Owners are also members members = members_team.user_set.exclude( - username__in=[ - user.username for user in owners]) + username__in=[user.username for user in owners] + ) # Exclude new owner if in members members = members.exclude(username=owner.username) # Add permissions to all users in Owners and Members team - [OwnerRole.add(owner, instance) for owner in owners] - [ReadOnlyRole.add(member, instance) for member in members] + for owner in owners: + OwnerRole.add(owner, instance) + for member in members: + ReadOnlyRole.add(member, instance) # clear cache - safe_delete('{}{}'.format(PROJ_PERM_CACHE, instance.pk)) + safe_delete(f"{PROJ_PERM_CACHE}{instance.pk}") - project = super(ProjectSerializer, self)\ - .update(instance, validated_data) + project = super().update(instance, validated_data) - project.xform_set.exclude(shared=project.shared)\ - .update(shared=project.shared, shared_data=project.shared) + project.xform_set.exclude(shared=project.shared).update( + shared=project.shared, shared_data=project.shared + ) return instance @track_object_event( - user_field='created_by', + user_field="created_by", properties={ - 'created_by': 'created_by', - 'project_id': 'pk', - 'project_name': 'name'} + "created_by": "created_by", + "project_id": "pk", + "project_name": "name", + }, ) def create(self, validated_data): - metadata = validated_data.get('metadata', dict()) + """Creates a project.""" + metadata = validated_data.get("metadata", {}) if metadata is None: - metadata = dict() - created_by = self.context['request'].user + metadata = {} + created_by = self.context["request"].user try: - project = Project.objects.create( # pylint: disable=E1101 - name=validated_data.get('name'), - organization=validated_data.get('organization'), + project = Project.objects.create( # pylint: disable=no-member + name=validated_data.get("name"), + organization=validated_data.get("organization"), created_by=created_by, - shared=validated_data.get('shared', False), - metadata=metadata) - except IntegrityError: + shared=validated_data.get("shared", False), + metadata=metadata, + ) + except IntegrityError as e: raise serializers.ValidationError( - "The fields name, organization must make a unique set.") + "The fields name, organization must make a unique set." + ) from e else: - project.xform_set.exclude(shared=project.shared)\ - .update(shared=project.shared, shared_data=project.shared) - request = self.context.get('request') - serializer = ProjectSerializer( - project, context={'request': request}) + project.xform_set.exclude(shared=project.shared).update( + shared=project.shared, shared_data=project.shared + ) + request = self.context.get("request") + serializer = ProjectSerializer(project, context={"request": request}) response = serializer.data - cache.set(f'{PROJ_OWNER_CACHE}{project.pk}', response) + cache.set(f"{PROJ_OWNER_CACHE}{project.pk}", response) return project def get_users(self, obj): # pylint: disable=no-self-use @@ -494,15 +569,17 @@ def get_forms(self, obj): # pylint: disable=no-self-use """ Return list of xforms in the project. """ - forms = cache.get('{}{}'.format(PROJ_FORMS_CACHE, obj.pk)) + project_forms_cache_key = f"{PROJ_FORMS_CACHE}{obj.pk}" + forms = cache.get(project_forms_cache_key) if forms: return forms xforms = get_project_xforms(obj) - request = self.context.get('request') + request = self.context.get("request") serializer = ProjectXFormSerializer( - xforms, context={'request': request}, many=True) + xforms, context={"request": request}, many=True + ) forms = list(serializer.data) - cache.set('{}{}'.format(PROJ_FORMS_CACHE, obj.pk), forms) + cache.set(project_forms_cache_key, forms) return forms @@ -522,7 +599,7 @@ def get_starred(self, obj): # pylint: disable=no-self-use """ Return True if request user has starred this project. """ - return is_starred(obj, self.context['request']) + return is_starred(obj, self.context["request"]) def get_teams(self, obj): # pylint: disable=no-self-use """ @@ -535,18 +612,22 @@ def get_data_views(self, obj): """ Return a list of filtered datasets. """ - data_views = cache.get('{}{}'.format(PROJECT_LINKED_DATAVIEWS, obj.pk)) + project_dataview_cache_key = f"{PROJECT_LINKED_DATAVIEWS}{obj.pk}" + data_views = cache.get(project_dataview_cache_key) if data_views: return data_views - data_views_obj = obj.dataview_prefetch if \ - hasattr(obj, 'dataview_prefetch') else\ - obj.dataview_set.filter(deleted_at__isnull=True) + data_views_obj = ( + obj.dataview_prefetch + if hasattr(obj, "dataview_prefetch") + else obj.dataview_set.filter(deleted_at__isnull=True) + ) serializer = DataViewMinimalSerializer( - data_views_obj, many=True, context=self.context) + data_views_obj, many=True, context=self.context + ) data_views = list(serializer.data) - cache.set('{}{}'.format(PROJECT_LINKED_DATAVIEWS, obj.pk), data_views) + cache.set(project_dataview_cache_key, data_views) return data_views diff --git a/onadata/libs/serializers/share_project_serializer.py b/onadata/libs/serializers/share_project_serializer.py index 33118910fa..342d91eea3 100644 --- a/onadata/libs/serializers/share_project_serializer.py +++ b/onadata/libs/serializers/share_project_serializer.py @@ -1,5 +1,5 @@ from django.contrib.auth.models import User -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import serializers from onadata.libs.models.share_project import ShareProject diff --git a/onadata/libs/serializers/share_team_project_serializer.py b/onadata/libs/serializers/share_team_project_serializer.py index bfedafc484..780be95304 100644 --- a/onadata/libs/serializers/share_team_project_serializer.py +++ b/onadata/libs/serializers/share_team_project_serializer.py @@ -1,52 +1,68 @@ -from django.utils.translation import ugettext as _ +# -*- coding: utf-8 -*- +""" +Share projects to team functions. +""" +from django.utils.translation import gettext as _ from rest_framework import serializers + from onadata.libs.models.share_team_project import ShareTeamProject from onadata.libs.permissions import ROLES -from onadata.libs.serializers.fields.team_field import TeamField from onadata.libs.serializers.fields.project_field import ProjectField +from onadata.libs.serializers.fields.team_field import TeamField class ShareTeamProjectSerializer(serializers.Serializer): + """Shares a project with a team.""" + team = TeamField() project = ProjectField() role = serializers.CharField(max_length=50) + # pylint: disable=no-self-use def update(self, instance, validated_data): - instance.team = validated_data.get('team', instance.team) - instance.project = validated_data.get('project', instance.project) - instance.role = validated_data.get('role', instance.role) + """Update project sharing properties.""" + instance.team = validated_data.get("team", instance.team) + instance.project = validated_data.get("project", instance.project) + instance.role = validated_data.get("role", instance.role) instance.save() return instance + # pylint: disable=no-self-use def create(self, validated_data): + """Shares a project to a team.""" instance = ShareTeamProject(**validated_data) instance.save() return instance + # pylint: disable=no-self-use def validate_role(self, value): """check that the role exists""" if value not in ROLES: - raise serializers.ValidationError(_( - u"Unknown role '%(role)s'." % {"role": value} - )) + raise serializers.ValidationError(_(f"Unknown role '{value}'.")) return value class RemoveTeamFromProjectSerializer(ShareTeamProjectSerializer): + """Remove a team from a project.""" + remove = serializers.BooleanField() + # pylint: disable=no-self-use def update(self, instance, validated_data): - instance.remove = validated_data.get('remove', instance.remove) + """Remove a team from a project""" + instance.remove = validated_data.get("remove", instance.remove) instance.save() return instance + # pylint: disable=no-self-use def create(self, validated_data): + """Remove a team from a project""" instance = ShareTeamProject(**validated_data) instance.save() diff --git a/onadata/libs/serializers/share_xform_serializer.py b/onadata/libs/serializers/share_xform_serializer.py index a9b65d72ee..c1c0f9e9e9 100644 --- a/onadata/libs/serializers/share_xform_serializer.py +++ b/onadata/libs/serializers/share_xform_serializer.py @@ -1,47 +1,60 @@ -from django.contrib.auth.models import User -from django.utils.translation import ugettext as _ +# -*- coding: utf-8 -*- +""" +Share XForm serializer. +""" +from django.contrib.auth import get_user_model +from django.utils.translation import gettext as _ from rest_framework import serializers + from onadata.libs.models.share_xform import ShareXForm from onadata.libs.permissions import ROLES from onadata.libs.serializers.fields.xform_field import XFormField class ShareXFormSerializer(serializers.Serializer): + """Share xform to a user.""" + xform = XFormField() username = serializers.CharField(max_length=255) role = serializers.CharField(max_length=50) + # pylint: disable=unused-argument,no-self-use def update(self, instance, validated_data): - instance.xform = validated_data.get('xform', instance.xform) - instance.username = validated_data.get('username', instance.username) - instance.role = validated_data.get('role', instance.role) + """Make changes to form share to a user.""" + instance.xform = validated_data.get("xform", instance.xform) + instance.username = validated_data.get("username", instance.username) + instance.role = validated_data.get("role", instance.role) instance.save() return instance + # pylint: disable=unused-argument,no-self-use def create(self, validated_data): + """Assign role permission for a form to a user.""" instance = ShareXForm(**validated_data) instance.save() return instance + # pylint: disable=no-self-use def validate_username(self, value): """Check that the username exists""" + # pylint: disable=invalid-name + User = get_user_model() # noqa N806 try: User.objects.get(username=value) - except User.DoesNotExist: - raise serializers.ValidationError(_( - u"User '%(value)s' does not exist." % {"value": value} - )) + except User.DoesNotExist as e: + raise serializers.ValidationError( + _(f"User '{value}' does not exist.") + ) from e return value + # pylint: disable=no-self-use def validate_role(self, value): """check that the role exists""" if value not in ROLES: - raise serializers.ValidationError(_( - u"Unknown role '%(role)s'." % {"role": value} - )) + raise serializers.ValidationError(_(f"Unknown role '{value}'.")) return value diff --git a/onadata/libs/serializers/stats_serializer.py b/onadata/libs/serializers/stats_serializer.py index ac329bd0eb..2db6c85f6f 100644 --- a/onadata/libs/serializers/stats_serializer.py +++ b/onadata/libs/serializers/stats_serializer.py @@ -1,4 +1,8 @@ -from django.utils.translation import ugettext as _ +# -*- coding: utf-8 -*- +""" +Stats API endpoint serializer. +""" +from django.utils.translation import gettext as _ from django.core.cache import cache from django.conf import settings @@ -6,108 +10,125 @@ from rest_framework import serializers from rest_framework.utils.serializer_helpers import ReturnList -from onadata.libs.data.statistics import\ - get_median_for_numeric_fields_in_form,\ - get_mean_for_numeric_fields_in_form,\ - get_mode_for_numeric_fields_in_form, get_min_max_range, get_all_stats +from onadata.libs.data.statistics import ( + get_median_for_numeric_fields_in_form, + get_mean_for_numeric_fields_in_form, + get_mode_for_numeric_fields_in_form, + get_min_max_range, + get_all_stats, +) from onadata.apps.logger.models.xform import XForm from onadata.libs.data.query import get_form_submissions_grouped_by_field from onadata.libs.utils.cache_tools import XFORM_SUBMISSION_STAT -SELECT_FIELDS = ['select one', 'select multiple'] +SELECT_FIELDS = ["select one", "select multiple"] STATS_FUNCTIONS = { - 'mean': get_mean_for_numeric_fields_in_form, - 'median': get_median_for_numeric_fields_in_form, - 'mode': get_mode_for_numeric_fields_in_form, - 'range': get_min_max_range + "mean": get_mean_for_numeric_fields_in_form, + "median": get_median_for_numeric_fields_in_form, + "mode": get_mode_for_numeric_fields_in_form, + "range": get_min_max_range, } class SubmissionStatsSerializer(serializers.HyperlinkedModelSerializer): + """Submission stats serializer for use with the list API endpoint, summary of the + submission stats endpoints.""" + url = serializers.HyperlinkedIdentityField( - view_name='submissionstats-detail', lookup_field='pk') + view_name="submissionstats-detail", lookup_field="pk" + ) class Meta: model = XForm - fields = ('id', 'id_string', 'url') + fields = ("id", "id_string", "url") +# pylint: disable=abstract-method class SubmissionStatsInstanceSerializer(serializers.Serializer): - def to_representation(self, obj): - if obj is None: - return super(SubmissionStatsInstanceSerializer, self)\ - .to_representation(obj) + """Submissions stats instance serializer - provides submission summary stats.""" + + def to_representation(self, instance): + """Returns submissions stats grouped by a specified field.""" + if instance is None: + return super().to_representation(instance) - request = self.context.get('request') - field = request.query_params.get('group') - name = request.query_params.get('name', field) + request = self.context.get("request") + field = request.query_params.get("group") + name = request.query_params.get("name", field) if field is None: - raise exceptions.ParseError(_(u"Expecting `group` and `name`" - u" query parameters.")) + raise exceptions.ParseError( + _("Expecting `group` and `name`" " query parameters.") + ) - cache_key = '{}{}{}{}'.format(XFORM_SUBMISSION_STAT, obj.pk, - field, name) + cache_key = f"{XFORM_SUBMISSION_STAT}{instance.pk}{field}{name}" data = cache.get(cache_key) if data: return data try: - data = get_form_submissions_grouped_by_field( - obj, field, name) + data = get_form_submissions_grouped_by_field(instance, field, name) except ValueError as e: raise exceptions.ParseError(detail=e) else: if data: - element = obj.get_survey_element(field) + element = instance.get_survey_element(field) if element and element.type in SELECT_FIELDS: for record in data: - label = obj.get_choice_label(element, record[name]) + label = instance.get_choice_label(element, record[name]) record[name] = label - cache.set(cache_key, data, - settings.XFORM_SUBMISSION_STAT_CACHE_TIME) + cache.set(cache_key, data, settings.XFORM_SUBMISSION_STAT_CACHE_TIME) return data @property def data(self): + """Return the data as a list with ReturnList instead of a python object.""" ret = super(serializers.Serializer, self).data return ReturnList(ret, serializer=self) class StatsSerializer(serializers.HyperlinkedModelSerializer): + """Stats serializer for use with the list API endpoint, summary of the stats + endpoints.""" + url = serializers.HyperlinkedIdentityField( - view_name='stats-detail', lookup_field='pk') + view_name="stats-detail", lookup_field="pk" + ) class Meta: model = XForm - fields = ('id', 'id_string', 'url') + fields = ("id", "id_string", "url") +# pylint: disable=abstract-method class StatsInstanceSerializer(serializers.Serializer): - def to_representation(self, obj): - if obj is None: - return super(StatsInstanceSerializer, self).to_representation(obj) + """The stats instance serializer - calls the relevant statistical functions and + returns the results against form data submissions.""" + + def to_representation(self, instance): + """Returns the result of the selected stats function.""" + if instance is None: + return super().to_representation(instance) - request = self.context.get('request') - method = request.query_params.get('method', None) - field = request.query_params.get('field', None) + request = self.context.get("request") + method = request.query_params.get("method", None) + field = request.query_params.get("field", None) - if field and field not in obj.get_keys(): + if field and field not in instance.get_keys(): raise exceptions.ParseError(detail=_("Field not in XForm.")) - stats_function = STATS_FUNCTIONS.get(method and method.lower(), - get_all_stats) + stats_function = STATS_FUNCTIONS.get(method and method.lower(), get_all_stats) try: - data = stats_function(obj, field) + data = stats_function(instance, field) except ValueError as e: raise exceptions.ParseError(detail=e) diff --git a/onadata/libs/serializers/tag_list_serializer.py b/onadata/libs/serializers/tag_list_serializer.py index 16ae5cdf13..ed6d184b0e 100644 --- a/onadata/libs/serializers/tag_list_serializer.py +++ b/onadata/libs/serializers/tag_list_serializer.py @@ -1,21 +1,28 @@ -from django.utils.translation import ugettext as _ +# -*- coding: utf-8 -*- +""" +Tags list serializer module. +""" +from django.utils.translation import gettext as _ from rest_framework import serializers class TagListSerializer(serializers.Field): + """Tags serializer - represents tags as a list of strings.""" def to_internal_value(self, data): - if not isinstance(data, list): - raise serializers.ValidationError(_(u"expected a list of data")) + """Validates the data is a list.""" + if isinstance(data, list): + return data - return data + raise serializers.ValidationError(_("expected a list of data")) - def to_representation(self, obj): - if obj is None: - return super(TagListSerializer, self).to_representation(obj) + def to_representation(self, value): + """Returns all tags linked to the object ``value`` as a list.""" + if value is None: + return super().to_representation(value) - if not isinstance(obj, list): - return [tag.name for tag in obj.all()] + if not isinstance(value, list): + return [tag.name for tag in value.all()] - return obj + return value diff --git a/onadata/libs/serializers/user_profile_serializer.py b/onadata/libs/serializers/user_profile_serializer.py index f2686c4131..8172a3fb6a 100644 --- a/onadata/libs/serializers/user_profile_serializer.py +++ b/onadata/libs/serializers/user_profile_serializer.py @@ -5,39 +5,39 @@ import copy import re -from past.builtins import basestring # pylint: disable=redefined-builtin - from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.contrib.sites.models import Site from django.core.cache import cache from django.db import IntegrityError, transaction -from django.utils.translation import ugettext as _ +from django.db.models.query import QuerySet from django.utils import timezone +from django.utils.translation import gettext as _ import six from django_digest.backend.db import update_partial_digests -from django.db.models.query import QuerySet from registration.models import RegistrationProfile from rest_framework import serializers -from onadata.apps.api.tasks import send_verification_email from onadata.apps.api.models.temp_token import TempToken +from onadata.apps.api.tasks import send_verification_email from onadata.apps.main.forms import RegistrationFormUserProfile from onadata.apps.main.models import UserProfile from onadata.libs.authentication import expired from onadata.libs.permissions import CAN_VIEW_PROFILE, is_organization from onadata.libs.serializers.fields.json_field import JsonField -from onadata.libs.utils.cache_tools import IS_ORG from onadata.libs.utils.analytics import track_object_event -from onadata.libs.utils.email import ( - get_verification_url, get_verification_email_data -) +from onadata.libs.utils.cache_tools import IS_ORG +from onadata.libs.utils.email import get_verification_email_data, get_verification_url RESERVED_NAMES = RegistrationFormUserProfile.RESERVED_USERNAMES LEGAL_USERNAMES_REGEX = RegistrationFormUserProfile.legal_usernames_re +# pylint: disable=invalid-name +User = get_user_model() + + def _get_first_last_names(name, limit=30): if not isinstance(name, six.string_types): return name, name @@ -47,59 +47,56 @@ def _get_first_last_names(name, limit=30): # imposition of 30 characters on both first_name and last_name hence # ensure we only have 30 characters for either field - return name[:limit], name[limit:limit * 2] + end = limit * 2 + return name[:limit], name[limit:end] name_split = name.split() first_name = name_split[0] - last_name = u'' + last_name = "" - if len(name_split) > 1: - last_name = u' '.join(name_split[1:]) + if name_split: + last_name = " ".join(name_split[1:]) return first_name, last_name def _get_registration_params(attrs): params = copy.deepcopy(attrs) - name = params.get('name', None) - user = params.pop('user', None) + name = params.get("name", None) + user = params.pop("user", None) if user: - username = user.pop('username', None) - password = user.pop('password', None) - first_name = user.pop('first_name', None) - last_name = user.pop('last_name', None) - email = user.pop('email', None) + username = user.pop("username", None) + password = user.pop("password", None) + first_name = user.pop("first_name", None) + last_name = user.pop("last_name", None) + email = user.pop("email", None) if username: - params['username'] = username + params["username"] = username if email: - params['email'] = email + params["email"] = email if password: - params.update({'password1': password, 'password2': password}) + params.update({"password1": password, "password2": password}) if first_name: - params['first_name'] = first_name + params["first_name"] = first_name - params['last_name'] = last_name or '' + params["last_name"] = last_name or "" # For backward compatibility, Users who still use only name if name: - first_name, last_name = \ - _get_first_last_names(name) - params['first_name'] = first_name - params['last_name'] = last_name + first_name, last_name = _get_first_last_names(name) + params["first_name"] = first_name + params["last_name"] = last_name return params def _send_verification_email(redirect_url, user, request): - verification_key = (user.registrationprofile - .create_new_activation_key()) - verification_url = get_verification_url( - redirect_url, request, verification_key - ) + verification_key = user.registrationprofile.create_new_activation_key() + verification_url = get_verification_url(redirect_url, request, verification_key) email_data = get_verification_email_data( user.email, user.username, verification_url, request @@ -112,47 +109,74 @@ class UserProfileSerializer(serializers.HyperlinkedModelSerializer): """ UserProfile serializer. """ + url = serializers.HyperlinkedIdentityField( - view_name='userprofile-detail', lookup_field='user') + view_name="userprofile-detail", lookup_field="user" + ) is_org = serializers.SerializerMethodField() - username = serializers.CharField(source='user.username', min_length=3, - max_length=30) + username = serializers.CharField( + source="user.username", min_length=3, max_length=30 + ) name = serializers.CharField(required=False, allow_blank=True) - first_name = serializers.CharField(source='user.first_name', - required=False, allow_blank=True, - max_length=30) - last_name = serializers.CharField(source='user.last_name', - required=False, allow_blank=True, - max_length=30) - email = serializers.EmailField(source='user.email') - website = serializers.CharField(source='home_page', required=False, - allow_blank=True) + first_name = serializers.CharField( + source="user.first_name", required=False, allow_blank=True, max_length=30 + ) + last_name = serializers.CharField( + source="user.last_name", required=False, allow_blank=True, max_length=30 + ) + email = serializers.EmailField(source="user.email") + website = serializers.CharField( + source="home_page", required=False, allow_blank=True + ) twitter = serializers.CharField(required=False, allow_blank=True) gravatar = serializers.ReadOnlyField() - password = serializers.CharField(source='user.password', allow_blank=True, - required=False) + password = serializers.CharField( + source="user.password", allow_blank=True, required=False + ) user = serializers.HyperlinkedRelatedField( - view_name='user-detail', lookup_field='username', read_only=True) + view_name="user-detail", lookup_field="username", read_only=True + ) metadata = JsonField(required=False) - id = serializers.ReadOnlyField(source='user.id') # pylint: disable=C0103 - joined_on = serializers.ReadOnlyField(source='user.date_joined') + # pylint: disable=invalid-name + id = serializers.ReadOnlyField(source="user.id") + joined_on = serializers.ReadOnlyField(source="user.date_joined") + # pylint: disable=too-few-public-methods,missing-class-docstring class Meta: model = UserProfile - fields = ('id', 'is_org', 'url', 'username', 'password', 'first_name', - 'last_name', 'email', 'city', 'country', 'organization', - 'website', 'twitter', 'gravatar', 'require_auth', 'user', - 'metadata', 'joined_on', 'name') - owner_only_fields = ('metadata', ) + fields = ( + "id", + "is_org", + "url", + "username", + "password", + "first_name", + "last_name", + "email", + "city", + "country", + "organization", + "website", + "twitter", + "gravatar", + "require_auth", + "user", + "metadata", + "joined_on", + "name", + ) + owner_only_fields = ("metadata",) def __init__(self, *args, **kwargs): - super(UserProfileSerializer, self).__init__(*args, **kwargs) - if self.instance and hasattr(self.Meta, 'owner_only_fields'): - request = self.context.get('request') - if isinstance(self.instance, QuerySet) or \ - (request and request.user != self.instance.user) or \ - not request: - for field in getattr(self.Meta, 'owner_only_fields'): + super().__init__(*args, **kwargs) + if self.instance and hasattr(self.Meta, "owner_only_fields"): + request = self.context.get("request") + if ( + isinstance(self.instance, QuerySet) + or (request and request.user != self.instance.user) + or not request + ): + for field in getattr(self.Meta, "owner_only_fields"): self.fields.pop(field) def get_is_org(self, obj): # pylint: disable=no-self-use @@ -160,59 +184,60 @@ def get_is_org(self, obj): # pylint: disable=no-self-use Returns True if it is an organization profile. """ if obj: - is_org = cache.get('{}{}'.format(IS_ORG, obj.pk)) + is_org = cache.get(f"{IS_ORG}{obj.pk}") if is_org: return is_org is_org = is_organization(obj) - cache.set('{}{}'.format(IS_ORG, obj.pk), is_org) + cache.set(f"{IS_ORG}{obj.pk}", is_org) return is_org def to_representation(self, instance): """ Serialize objects -> primitives. """ - ret = super(UserProfileSerializer, self).to_representation(instance) - if 'password' in ret: - del ret['password'] + ret = super().to_representation(instance) + if "password" in ret: + del ret["password"] - request = self.context['request'] \ - if 'request' in self.context else None + request = self.context["request"] if "request" in self.context else None - if 'email' in ret and request is None or request.user \ - and not request.user.has_perm(CAN_VIEW_PROFILE, instance): - del ret['email'] + if ( + "email" in ret + and request is None + or request.user + and not request.user.has_perm(CAN_VIEW_PROFILE, instance) + ): + del ret["email"] - if 'first_name' in ret: - ret['name'] = u' '.join([ret.get('first_name'), - ret.get('last_name', "")]) - ret['name'] = ret['name'].strip() + if "first_name" in ret: + ret["name"] = " ".join([ret.get("first_name"), ret.get("last_name", "")]) + ret["name"] = ret["name"].strip() return ret def update(self, instance, validated_data): + """Update user properties.""" params = validated_data password = params.get("password1") - email = params.get('email') + email = params.get("email") # Check password if email is being updated if email and not password: raise serializers.ValidationError( - _(u'Your password is required when updating your email ' - u'address.')) + _("Your password is required when updating your email address.") + ) if password and not instance.user.check_password(password): - raise serializers.ValidationError(_(u'Invalid password')) + raise serializers.ValidationError(_("Invalid password")) # get user instance.user.email = email or instance.user.email - instance.user.first_name = params.get('first_name', - instance.user.first_name) + instance.user.first_name = params.get("first_name", instance.user.first_name) - instance.user.last_name = params.get('last_name', - instance.user.last_name) + instance.user.last_name = params.get("last_name", instance.user.last_name) - instance.user.username = params.get('username', instance.user.username) + instance.user.username = params.get("username", instance.user.username) instance.user.save() @@ -220,92 +245,91 @@ def update(self, instance, validated_data): instance.metadata.update({"is_email_verified": False}) instance.save() - request = self.context.get('request') - redirect_url = params.get('redirect_url') + request = self.context.get("request") + redirect_url = params.get("redirect_url") _send_verification_email(redirect_url, instance.user, request) if password: # force django-digest to regenerate its stored partial digests update_partial_digests(instance.user, password) - return super(UserProfileSerializer, self).update(instance, params) + return super().update(instance, params) @track_object_event( - user_field='user', - properties={ - 'name': 'name', - 'country': 'country'}) + user_field="user", properties={"name": "name", "country": "country"} + ) def create(self, validated_data): + """Creates a user registration profile and account.""" params = validated_data - request = self.context.get('request') + request = self.context.get("request") metadata = {} - + username = params.get("username") site = Site.objects.get(pk=settings.SITE_ID) try: new_user = RegistrationProfile.objects.create_inactive_user( - username=params.get('username'), - password=params.get('password1'), - email=params.get('email'), + username=username, + password=params.get("password1"), + email=params.get("email"), site=site, - send_email=settings.SEND_EMAIL_ACTIVATION_API) - except IntegrityError: - raise serializers.ValidationError(_( - u"User account {} already exists".format( - params.get('username')) - )) + send_email=settings.SEND_EMAIL_ACTIVATION_API, + ) + except IntegrityError as e: + raise serializers.ValidationError( + _(f"User account {username} already exists") + ) from e new_user.is_active = True - new_user.first_name = params.get('first_name') - new_user.last_name = params.get('last_name') + new_user.first_name = params.get("first_name") + new_user.last_name = params.get("last_name") new_user.save() - if getattr( - settings, 'ENABLE_EMAIL_VERIFICATION', False - ): - redirect_url = params.get('redirect_url') + if getattr(settings, "ENABLE_EMAIL_VERIFICATION", False): + redirect_url = params.get("redirect_url") _send_verification_email(redirect_url, new_user, request) created_by = request.user created_by = None if created_by.is_anonymous else created_by - metadata['last_password_edit'] = timezone.now().isoformat() + metadata["last_password_edit"] = timezone.now().isoformat() profile = UserProfile( - user=new_user, name=params.get('first_name'), + user=new_user, + name=params.get("first_name"), created_by=created_by, - city=params.get('city', u''), - country=params.get('country', u''), - organization=params.get('organization', u''), - home_page=params.get('home_page', u''), - twitter=params.get('twitter', u''), - metadata=metadata + city=params.get("city", ""), + country=params.get("country", ""), + organization=params.get("organization", ""), + home_page=params.get("home_page", ""), + twitter=params.get("twitter", ""), + metadata=metadata, ) profile.save() + return profile def validate_username(self, value): """ Validate username. """ - username = value.lower() if isinstance(value, basestring) else value + username = value.lower() if isinstance(value, str) else value if username in RESERVED_NAMES: - raise serializers.ValidationError(_( - u"%s is a reserved name, please choose another" % username - )) - elif not LEGAL_USERNAMES_REGEX.search(username): - raise serializers.ValidationError(_( - u"username may only contain alpha-numeric characters and " - u"underscores" - )) - elif len(username) < 3: - raise serializers.ValidationError(_( - u"Username must have 3 or more characters" - )) + raise serializers.ValidationError( + _(f"{username} is a reserved name, please choose another") + ) + if not LEGAL_USERNAMES_REGEX.search(username): + raise serializers.ValidationError( + _( + "username may only contain alpha-numeric characters and " + "underscores" + ) + ) + if len(username) < 3: + raise serializers.ValidationError( + _("Username must have 3 or more characters") + ) users = User.objects.filter(username=username) if self.instance: users = users.exclude(pk=self.instance.user.pk) if users.exists(): - raise serializers.ValidationError(_( - u"%s already exists" % username - )) + raise serializers.ValidationError(_(f"{username} already exists")) return username @@ -318,9 +342,9 @@ def validate_email(self, value): users = users.exclude(pk=self.instance.user.pk) if users.exists(): - raise serializers.ValidationError(_( - u"This email address is already in use. " - )) + raise serializers.ValidationError( + _("This email address is already in use. ") + ) return value @@ -328,22 +352,25 @@ def validate_twitter(self, value): # pylint: disable=no-self-use """ Checks if the twitter handle is valid. """ - if isinstance(value, basestring) and value: + if isinstance(value, str) and value: match = re.search(r"^[A-Za-z0-9_]{1,15}$", value) if not match: - raise serializers.ValidationError(_( - u"Invalid twitter username {}".format(value) - )) + raise serializers.ValidationError( + _(f"Invalid twitter username {value}") + ) return value def validate(self, attrs): params = _get_registration_params(attrs) - if not self.instance and params.get('name') is None and \ - params.get('first_name') is None: - raise serializers.ValidationError({ - 'name': _(u"Either name or first_name should be provided") - }) + if ( + not self.instance + and params.get("name") is None + and params.get("first_name") is None + ): + raise serializers.ValidationError( + {"name": _("Either name or first_name should be provided")} + ) return params @@ -352,42 +379,59 @@ class UserProfileWithTokenSerializer(serializers.HyperlinkedModelSerializer): """ User Profile Serializer that includes the users API Tokens. """ + url = serializers.HyperlinkedIdentityField( - view_name='userprofile-detail', - lookup_field='user') - username = serializers.CharField(source='user.username') - email = serializers.CharField(source='user.email') - website = serializers.CharField(source='home_page', required=False) + view_name="userprofile-detail", lookup_field="user" + ) + username = serializers.CharField(source="user.username") + email = serializers.CharField(source="user.email") + website = serializers.CharField(source="home_page", required=False) gravatar = serializers.ReadOnlyField() user = serializers.HyperlinkedRelatedField( - view_name='user-detail', lookup_field='username', read_only=True) + view_name="user-detail", lookup_field="username", read_only=True + ) api_token = serializers.SerializerMethodField() temp_token = serializers.SerializerMethodField() class Meta: model = UserProfile - fields = ('url', 'username', 'name', 'email', 'city', - 'country', 'organization', 'website', 'twitter', 'gravatar', - 'require_auth', 'user', 'api_token', 'temp_token') + fields = ( + "url", + "username", + "name", + "email", + "city", + "country", + "organization", + "website", + "twitter", + "gravatar", + "require_auth", + "user", + "api_token", + "temp_token", + ) - def get_api_token(self, object): # pylint: disable=R0201,W0622 + # pylint: disable=no-self-use + def get_api_token(self, obj): """ Returns user's API Token. """ - return object.user.auth_token.key + return obj.user.auth_token.key - def get_temp_token(self, object): # pylint: disable=R0201,W0622 + # pylint: disable=no-self-use + def get_temp_token(self, obj): """ This should return a valid temp token for this user profile. """ - token, created = TempToken.objects.get_or_create(user=object.user) - check_expired = getattr(settings, 'CHECK_EXPIRED_TEMP_TOKEN', True) + token, created = TempToken.objects.get_or_create(user=obj.user) + check_expired = getattr(settings, "CHECK_EXPIRED_TEMP_TOKEN", True) try: if check_expired and not created and expired(token.created): with transaction.atomic(): - TempToken.objects.get(user=object.user).delete() - token = TempToken.objects.create(user=object.user) + TempToken.objects.get(user=obj.user).delete() + token = TempToken.objects.create(user=obj.user) except IntegrityError: pass diff --git a/onadata/libs/serializers/widget_serializer.py b/onadata/libs/serializers/widget_serializer.py index e93fb24ce9..88d1ac63ae 100644 --- a/onadata/libs/serializers/widget_serializer.py +++ b/onadata/libs/serializers/widget_serializer.py @@ -1,8 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Widget serializer +""" from django.core.exceptions import ObjectDoesNotExist from django.http import Http404 from django.urls import resolve, get_script_prefix, Resolver404 -from django.utils.translation import ugettext as _ -from future.moves.urllib.parse import urlparse +from django.utils.translation import gettext as _ +from six.moves.urllib.parse import urlparse from guardian.shortcuts import get_users_with_perms from rest_framework import serializers @@ -18,74 +22,91 @@ class GenericRelatedField(serializers.HyperlinkedRelatedField): + """ + GenericRelatedField - handle related field relations for XForm and DataView + """ + default_error_messages = { - 'incorrect_match': _('`{input}` is not a valid relation.') + "incorrect_match": _("`{input}` is not a valid relation.") } def __init__(self, *args, **kwargs): - self.view_names = ['xform-detail', 'dataviews-detail'] + self.view_names = ["xform-detail", "dataviews-detail"] self.resolve = resolve self.reverse = reverse - self.format = kwargs.pop('format', 'json') + self.format = kwargs.pop("format", "json") + # pylint: disable=bad-super-call super(serializers.RelatedField, self).__init__(*args, **kwargs) def _setup_field(self, view_name): + # pylint: disable=attribute-defined-outside-init self.lookup_url_kwarg = self.lookup_field - if view_name == 'xform-detail': + if view_name == "xform-detail": + # pylint: disable=attribute-defined-outside-init self.queryset = XForm.objects.all() - if view_name == 'dataviews-detail': + if view_name == "dataviews-detail": + # pylint: disable=attribute-defined-outside-init self.queryset = DataView.objects.all() def to_representation(self, value): + """Set's the self.view_name based on the type of ``value``.""" if isinstance(value, XForm): - self.view_name = 'xform-detail' + # pylint: disable=attribute-defined-outside-init + self.view_name = "xform-detail" elif isinstance(value, DataView): - self.view_name = 'dataviews-detail' + # pylint: disable=attribute-defined-outside-init + self.view_name = "dataviews-detail" else: - raise Exception(_(u"Unknown type for content_object.")) + raise Exception(_("Unknown type for content_object.")) self._setup_field(self.view_name) + # pylint: disable=bad-super-call return super(GenericRelatedField, self).to_representation(value) def to_internal_value(self, data): + """Verifies that ``data`` is a valid URL.""" try: - http_prefix = data.startswith(('http:', 'https:')) + http_prefix = data.startswith(("http:", "https:")) except AttributeError: - self.fail('incorrect_type', data_type=type(data).__name__) + self.fail("incorrect_type", data_type=type(data).__name__) input_data = data if http_prefix: # If needed convert absolute URLs to relative path data = urlparse(data).path prefix = get_script_prefix() if data.startswith(prefix): - data = '/' + data[len(prefix):] + data = "/" + data[len(prefix) :] try: match = self.resolve(data) except Resolver404: - self.fail('no_match') + self.fail("no_match") if match.view_name not in self.view_names: - self.fail('incorrect_match', input=input_data) + self.fail("incorrect_match", input=input_data) self._setup_field(match.view_name) try: return self.get_object(match.view_name, match.args, match.kwargs) except (ObjectDoesNotExist, TypeError, ValueError): - self.fail('does_not_exist') + self.fail("does_not_exist") return data class WidgetSerializer(serializers.HyperlinkedModelSerializer): + """ + WidgetSerializer + """ + + # pylint: disable=invalid-name id = serializers.ReadOnlyField() url = serializers.HyperlinkedIdentityField( - view_name='widgets-detail', - lookup_field='pk' + view_name="widgets-detail", lookup_field="pk" ) content_object = GenericRelatedField() key = serializers.CharField(read_only=True) @@ -93,19 +114,40 @@ class WidgetSerializer(serializers.HyperlinkedModelSerializer): order = serializers.IntegerField(required=False) metadata = JsonField(required=False) + # pylint: disable=too-few-public-methods class Meta: + """ + Meta model - specifies the fields in the Model Widget for the serializer + """ + model = Widget - fields = ('id', 'url', 'key', 'title', 'description', 'widget_type', - 'order', 'view_type', 'column', 'group_by', 'content_object', - 'data', 'aggregation', 'metadata') + fields = ( + "id", + "url", + "key", + "title", + "description", + "widget_type", + "order", + "view_type", + "column", + "group_by", + "content_object", + "data", + "aggregation", + "metadata", + ) def get_data(self, obj): + """ + Return the Widget.query_data(obj) + """ # Get the request obj - request = self.context.get('request') + request = self.context.get("request") # Check if data flag is present - data_flag = request.GET.get('data') - key = request.GET.get('key') + data_flag = request.GET.get("data") + key = request.GET.get("key") if (str2bool(data_flag) or key) and obj: data = Widget.query_data(obj) @@ -115,11 +157,12 @@ def get_data(self, obj): return data def validate(self, attrs): - column = attrs.get('column') + """Validates that column exists in the XForm.""" + column = attrs.get("column") # Get the form - if 'content_object' in attrs: - content_object = attrs.get('content_object') + if "content_object" in attrs: + content_object = attrs.get("content_object") if isinstance(content_object, XForm): xform = content_object @@ -130,12 +173,12 @@ def validate(self, attrs): try: # Check if column exists in xform get_field_from_field_xpath(column, xform) - except Http404: - raise serializers.ValidationError({ - 'column': (u"'{}' not in the form.".format(column)) - }) + except Http404 as e: + raise serializers.ValidationError( + {"column": f"'{column}' not in the form."} + ) from e - order = attrs.get('order') + order = attrs.get("order") # Set the order if order: @@ -144,19 +187,20 @@ def validate(self, attrs): return attrs def validate_content_object(self, value): - request = self.context.get('request') + """ + Validate if a user is the owner f the organization. + """ + request = self.context.get("request") users = get_users_with_perms( value.project, attach_perms=False, with_group_users=False ) profile = value.project.organization.profile # Shared or an admin in the organization - if request.user not in users and not\ - is_organization(profile) and not\ - OwnerRole.user_has_role(request.user, - profile): - raise serializers.ValidationError(_( - u"You don't have permission to the Project." - )) + is_owner = OwnerRole.user_has_role(request.user, profile) + if request.user not in users and not is_organization(profile) and not is_owner: + raise serializers.ValidationError( + _("You don't have permission to the Project.") + ) return value diff --git a/onadata/libs/serializers/xform_serializer.py b/onadata/libs/serializers/xform_serializer.py index fcf5408172..41dac1a253 100644 --- a/onadata/libs/serializers/xform_serializer.py +++ b/onadata/libs/serializers/xform_serializer.py @@ -1,45 +1,54 @@ +# -*- coding: utf-8 -*- +""" +XForm model serialization. +""" +import hashlib import logging import os -from hashlib import md5 -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.serialization import load_pem_public_key +from six import itervalues +from six.moves.urllib.parse import urlparse + from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.cache import cache from django.core.exceptions import ValidationError from django.core.validators import URLValidator from django.db.models import Count -from django.utils.translation import ugettext as _ -from future.moves.urllib.parse import urlparse -from future.utils import listvalues -from requests.exceptions import ConnectionError +from django.utils.translation import gettext as _ + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.serialization import load_pem_public_key from rest_framework import serializers from rest_framework.reverse import reverse -from onadata.apps.logger.models import DataView, Instance, XForm +from onadata.apps.logger.models import DataView, Instance, XForm, XFormVersion from onadata.apps.main.models.meta_data import MetaData -from onadata.apps.logger.models import XFormVersion from onadata.libs.exceptions import EnketoError from onadata.libs.permissions import get_role, is_organization -from onadata.libs.serializers.dataview_serializer import \ - DataViewMinimalSerializer +from onadata.libs.serializers.dataview_serializer import DataViewMinimalSerializer from onadata.libs.serializers.metadata_serializer import MetaDataSerializer from onadata.libs.serializers.tag_list_serializer import TagListSerializer from onadata.libs.utils.cache_tools import ( - ENKETO_PREVIEW_URL_CACHE, ENKETO_URL_CACHE, ENKETO_SINGLE_SUBMIT_URL_CACHE, - XFORM_LINKED_DATAVIEWS, XFORM_METADATA_CACHE, XFORM_PERMISSIONS_CACHE, - XFORM_DATA_VERSIONS, XFORM_COUNT) -from onadata.libs.utils.common_tags import (GROUP_DELIMETER_TAG, - REPEAT_INDEX_TAGS) + ENKETO_PREVIEW_URL_CACHE, + ENKETO_SINGLE_SUBMIT_URL_CACHE, + ENKETO_URL_CACHE, + XFORM_COUNT, + XFORM_DATA_VERSIONS, + XFORM_LINKED_DATAVIEWS, + XFORM_METADATA_CACHE, + XFORM_PERMISSIONS_CACHE, +) +from onadata.libs.utils.common_tags import GROUP_DELIMETER_TAG, REPEAT_INDEX_TAGS from onadata.libs.utils.decorators import check_obj -from onadata.libs.utils.viewer_tools import ( - get_enketo_urls, get_form_url) +from onadata.libs.utils.viewer_tools import get_enketo_urls, get_form_url +SUBMISSION_RETRIEVAL_THRESHOLD = getattr( + settings, "SUBMISSION_RETRIEVAL_THRESHOLD", 10000 +) -SUBMISSION_RETRIEVAL_THRESHOLD = getattr(settings, - "SUBMISSION_RETRIEVAL_THRESHOLD", - 10000) +# pylint: disable=invalid-name +User = get_user_model() def _create_enketo_urls(request, xform): @@ -50,9 +59,13 @@ def _create_enketo_urls(request, xform): :param xform: :return: enketo urls """ - form_url = get_form_url(request, xform.user.username, - settings.ENKETO_PROTOCOL, xform_pk=xform.pk, - generate_consistent_urls=True) + form_url = get_form_url( + request, + xform.user.username, + settings.ENKETO_PROTOCOL, + xform_pk=xform.pk, + generate_consistent_urls=True, + ) data = {} try: enketo_urls = get_enketo_urls(form_url, xform.id_string) @@ -60,20 +73,19 @@ def _create_enketo_urls(request, xform): return data offline_url = enketo_urls.get("offline_url") MetaData.enketo_url(xform, offline_url) - data['offline_url'] = offline_url - if 'preview_url' in enketo_urls: + data["offline_url"] = offline_url + if "preview_url" in enketo_urls: preview_url = enketo_urls.get("preview_url") MetaData.enketo_preview_url(xform, preview_url) - data['preview_url'] = preview_url - if 'single_url' in enketo_urls: + data["preview_url"] = preview_url + if "single_url" in enketo_urls: single_url = enketo_urls.get("single_url") - MetaData.enketo_single_submit_url( - xform, single_url) - data['single_url'] = single_url + MetaData.enketo_single_submit_url(xform, single_url) + data["single_url"] = single_url except ConnectionError as e: - logging.exception("Connection Error: %s" % e) + logging.exception("Connection Error: %s", e) except EnketoError as e: - logging.exception("Enketo Error: %s" % e.message) + logging.exception("Enketo Error: %s", e.message) return data @@ -87,55 +99,77 @@ def _set_cache(cache_key, cache_data, obj): :param obj: :return: Data that has been cached """ - cache.set('{}{}'.format(cache_key, obj.pk), cache_data) + cache.set(f"{cache_key}{obj.pk}", cache_data) return cache_data def user_to_username(item): - item['user'] = item['user'].username + """ + Replaces the item["user"] user object with the to user.username. + """ + item["user"] = item["user"].username return item def clean_public_key(value): - if value.startswith('-----BEGIN PUBLIC KEY-----') and\ - value.endswith('-----END PUBLIC KEY-----'): - return value.replace('-----BEGIN PUBLIC KEY-----', - '').replace('-----END PUBLIC KEY-----', - '').replace(' ', '').rstrip() + """ + Strips public key comments and spaces from a public key ``value``. + """ + if value.startswith("-----BEGIN PUBLIC KEY-----") and value.endswith( + "-----END PUBLIC KEY-----" + ): + return ( + value.replace("-----BEGIN PUBLIC KEY-----", "") + .replace("-----END PUBLIC KEY-----", "") + .replace(" ", "") + .rstrip() + ) + + return value +# pylint: disable=too-few-public-methods class MultiLookupIdentityField(serializers.HyperlinkedIdentityField): """ Custom HyperlinkedIdentityField that supports multiple lookup fields. Credits: https://stackoverflow.com/a/31161585 """ - lookup_fields = (('pk', 'pk'),) + + lookup_fields = (("pk", "pk"),) def __init__(self, *args, **kwargs): - self.lookup_fields = kwargs.pop('lookup_fields', self.lookup_fields) - super(MultiLookupIdentityField, self).__init__(*args, **kwargs) + self.lookup_fields = kwargs.pop("lookup_fields", self.lookup_fields) + super().__init__(*args, **kwargs) + # pylint: disable=redefined-builtin def get_url(self, obj, view_name, request, format): + """ + Returns URL to the given object. + """ kwargs = {} for model_field, url_param in self.lookup_fields: attr = obj - for field in model_field.split('__'): + for field in model_field.split("__"): attr = getattr(attr, field) kwargs[url_param] = attr - if not format and hasattr(self, 'format'): - fmt = self.format - else: - fmt = format + fmt = self.format if not format and hasattr(self, "format") else format + + return reverse(view_name, kwargs=kwargs, request=request, format=fmt) - return reverse( - view_name, kwargs=kwargs, request=request, format=fmt) +class XFormMixin: + """ + XForm mixins + """ -class XFormMixin(object): + # pylint: disable=no-self-use def get_xls_available(self, obj): + """ + Returns True if ``obj.xls.url`` is not None, indicates XLS is present. + """ available = False if obj and obj.xls: try: @@ -149,96 +183,102 @@ def _get_metadata(self, obj, key): for m in obj.metadata_set.all(): if m.data_type == key: return m.data_value - else: - return obj.metadata_set.all() + return None + + # pylint: disable=no-self-use def get_users(self, obj): + """ + Returns a list of users based on XForm permissions. + """ xform_perms = [] if obj: - xform_perms = cache.get( - '{}{}'.format(XFORM_PERMISSIONS_CACHE, obj.pk)) + xform_perms = cache.get(f"{XFORM_PERMISSIONS_CACHE}{obj.pk}") if xform_perms: return xform_perms - cache.set('{}{}'.format(XFORM_PERMISSIONS_CACHE, obj.pk), - xform_perms) + cache.set(f"{XFORM_PERMISSIONS_CACHE}{obj.pk}", xform_perms) data = {} for perm in obj.xformuserobjectpermission_set.all(): if perm.user_id not in data: user = perm.user data[perm.user_id] = { - 'permissions': [], - 'is_org': is_organization(user.profile), - 'metadata': user.profile.metadata, - 'first_name': user.first_name, - 'last_name': user.last_name, - 'user': user.username + "permissions": [], + "is_org": is_organization(user.profile), + "metadata": user.profile.metadata, + "first_name": user.first_name, + "last_name": user.last_name, + "user": user.username, } if perm.user_id in data: - data[perm.user_id]['permissions'].append( - perm.permission.codename) + data[perm.user_id]["permissions"].append(perm.permission.codename) for k in list(data): - data[k]['permissions'].sort() - data[k]['role'] = get_role(data[k]['permissions'], XForm) - del (data[k]['permissions']) + data[k]["permissions"].sort() + data[k]["role"] = get_role(data[k]["permissions"], XForm) + del data[k]["permissions"] - xform_perms = listvalues(data) + xform_perms = list(itervalues(data)) - cache.set('{}{}'.format(XFORM_PERMISSIONS_CACHE, obj.pk), xform_perms) + cache.set(f"{XFORM_PERMISSIONS_CACHE}{obj.pk}", xform_perms) return xform_perms def get_enketo_url(self, obj): + """ + Returns Enketo URL for given ``obj``. + """ if obj: - _enketo_url = cache.get('{}{}'.format(ENKETO_URL_CACHE, obj.pk)) + _enketo_url = cache.get(f"{ENKETO_URL_CACHE}{obj.pk}") if _enketo_url: return _enketo_url - url = self._get_metadata(obj, 'enketo_url') + url = self._get_metadata(obj, "enketo_url") if url is None: - enketo_urls = _create_enketo_urls( - self.context.get('request'), obj) - url = enketo_urls.get('offline_url') + enketo_urls = _create_enketo_urls(self.context.get("request"), obj) + url = enketo_urls.get("offline_url") return _set_cache(ENKETO_URL_CACHE, url, obj) return None def get_enketo_single_submit_url(self, obj): + """ + Returns single submit Enketo URL for given ``obj``. + """ if obj: _enketo_single_submit_url = cache.get( - '{}{}'.format(ENKETO_SINGLE_SUBMIT_URL_CACHE, - obj.pk)) + f"{ENKETO_SINGLE_SUBMIT_URL_CACHE}{obj.pk}" + ) if _enketo_single_submit_url: return _enketo_single_submit_url - url = self._get_metadata(obj, 'enketo_url') + url = self._get_metadata(obj, "enketo_url") if url is None: - enketo_urls = _create_enketo_urls( - self.context.get('request'), obj) - url = enketo_urls.get('offline_url') + enketo_urls = _create_enketo_urls(self.context.get("request"), obj) + url = enketo_urls.get("offline_url") - return _set_cache( - ENKETO_SINGLE_SUBMIT_URL_CACHE, url, obj) + return _set_cache(ENKETO_SINGLE_SUBMIT_URL_CACHE, url, obj) return None def get_enketo_preview_url(self, obj): + """ + Returns preview Enketo URL for given ``obj``. + """ if obj: - _enketo_preview_url = cache.get( - '{}{}'.format(ENKETO_PREVIEW_URL_CACHE, obj.pk)) + _enketo_preview_url = cache.get(f"{ENKETO_PREVIEW_URL_CACHE}{obj.pk}") if _enketo_preview_url: return _enketo_preview_url - url = self._get_metadata(obj, 'enketo_preview_url') + url = self._get_metadata(obj, "enketo_preview_url") if url is None: - try: - enketo_urls = _create_enketo_urls( - self.context.get('request'), obj) - url = enketo_urls.get('preview_url') - except Exception: + enketo_urls = _create_enketo_urls(self.context.get("request"), obj) + url = ( + enketo_urls["preview_url"] if "preview_url" in enketo_urls else url + ) + if url is None: return url return _set_cache(ENKETO_PREVIEW_URL_CACHE, url, obj) @@ -246,33 +286,40 @@ def get_enketo_preview_url(self, obj): return None def get_data_views(self, obj): + """Returns a list of filtered datasets linked to the form.""" if obj: - key = '{}{}'.format(XFORM_LINKED_DATAVIEWS, obj.pk) + key = f"{XFORM_LINKED_DATAVIEWS}{obj.pk}" data_views = cache.get(key) if data_views: return data_views data_views = DataViewMinimalSerializer( obj.dataview_set.filter(deleted_at__isnull=True), - many=True, context=self.context).data + many=True, + context=self.context, + ).data cache.set(key, list(data_views)) return data_views return [] + # pylint: disable=no-self-use def get_num_of_submissions(self, obj): + """ + Returns number of submissions. + """ if obj: - key = '{}{}'.format(XFORM_COUNT, obj.pk) + key = f"{XFORM_COUNT}{obj.pk}" count = cache.get(key) if count: return count - force_update = True if obj.is_merged_dataset else False - count = obj.submission_count(force_update) + count = obj.submission_count(obj.is_merged_dataset) cache.set(key, count) return count + return 0 def get_last_submission_time(self, obj): """Return datetime of last submission @@ -280,43 +327,51 @@ def get_last_submission_time(self, obj): If a form is a merged dataset then it is picked from the list of forms attached to that merged dataset. """ - if 'last_submission_time' not in self.fields: + if "last_submission_time" not in self.fields: return None if obj.is_merged_dataset: values = [ x.last_submission_time.isoformat() - for x in obj.mergedxform.xforms.only('last_submission_time') + for x in obj.mergedxform.xforms.only("last_submission_time") if x.last_submission_time ] if values: return sorted(values, reverse=True)[0] - return obj.last_submission_time.isoformat() \ - if obj.last_submission_time else None + return ( + obj.last_submission_time.isoformat() if obj.last_submission_time else None + ) class XFormBaseSerializer(XFormMixin, serializers.HyperlinkedModelSerializer): - formid = serializers.ReadOnlyField(source='id') + """XForm base serializer.""" + + formid = serializers.ReadOnlyField(source="id") owner = serializers.HyperlinkedRelatedField( - view_name='user-detail', - source='user', - lookup_field='username', + view_name="user-detail", + source="user", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME)) + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), + ) created_by = serializers.HyperlinkedRelatedField( - view_name='user-detail', - lookup_field='username', + view_name="user-detail", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME)) - public = serializers.BooleanField(source='shared') - public_data = serializers.BooleanField(source='shared_data') + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), + ) + public = serializers.BooleanField(source="shared") + public_data = serializers.BooleanField(source="shared_data") public_key = serializers.CharField(required=False) require_auth = serializers.BooleanField() tags = TagListSerializer(read_only=True) title = serializers.CharField(max_length=255) url = serializers.HyperlinkedIdentityField( - view_name='xform-detail', lookup_field='pk') + view_name="xform-detail", lookup_field="pk" + ) users = serializers.SerializerMethodField() enketo_url = serializers.SerializerMethodField() enketo_preview_url = serializers.SerializerMethodField() @@ -326,39 +381,65 @@ class XFormBaseSerializer(XFormMixin, serializers.HyperlinkedModelSerializer): data_views = serializers.SerializerMethodField() xls_available = serializers.SerializerMethodField() + # pylint: disable=too-few-public-methods,missing-class-docstring class Meta: model = XForm - read_only_fields = ('json', 'xml', 'date_created', 'date_modified', - 'encrypted', 'bamboo_dataset', - 'last_submission_time', 'is_merged_dataset', - 'xls_available') - exclude = ('json', 'xml', 'xls', 'user', 'has_start_time', 'shared', - 'shared_data', 'deleted_at', 'deleted_by') + read_only_fields = ( + "json", + "xml", + "date_created", + "date_modified", + "encrypted", + "bamboo_dataset", + "last_submission_time", + "is_merged_dataset", + "xls_available", + ) + exclude = ( + "json", + "xml", + "xls", + "user", + "has_start_time", + "shared", + "shared_data", + "deleted_at", + "deleted_by", + ) class XFormSerializer(XFormMixin, serializers.HyperlinkedModelSerializer): - formid = serializers.ReadOnlyField(source='id') + """ + XForm model serializer + """ + + formid = serializers.ReadOnlyField(source="id") metadata = serializers.SerializerMethodField() owner = serializers.HyperlinkedRelatedField( - view_name='user-detail', - source='user', - lookup_field='username', + view_name="user-detail", + source="user", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME)) + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), + ) created_by = serializers.HyperlinkedRelatedField( - view_name='user-detail', - lookup_field='username', + view_name="user-detail", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME)) - public = serializers.BooleanField(source='shared') - public_data = serializers.BooleanField(source='shared_data') + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), + ) + public = serializers.BooleanField(source="shared") + public_data = serializers.BooleanField(source="shared_data") public_key = serializers.CharField(required=False) require_auth = serializers.BooleanField() submission_count_for_today = serializers.ReadOnlyField() tags = TagListSerializer(read_only=True) title = serializers.CharField(max_length=255) url = serializers.HyperlinkedIdentityField( - view_name='xform-detail', lookup_field='pk') + view_name="xform-detail", lookup_field="pk" + ) users = serializers.SerializerMethodField() enketo_url = serializers.SerializerMethodField() enketo_preview_url = serializers.SerializerMethodField() @@ -371,27 +452,46 @@ class XFormSerializer(XFormMixin, serializers.HyperlinkedModelSerializer): class Meta: model = XForm - read_only_fields = ('json', 'xml', 'date_created', 'date_modified', - 'encrypted', 'bamboo_dataset', - 'last_submission_time', 'is_merged_dataset', - 'xls_available') - exclude = ('json', 'xml', 'xls', 'user', 'has_start_time', 'shared', - 'shared_data', 'deleted_at', 'deleted_by') - + read_only_fields = ( + "json", + "xml", + "date_created", + "date_modified", + "encrypted", + "bamboo_dataset", + "last_submission_time", + "is_merged_dataset", + "xls_available", + ) + exclude = ( + "json", + "xml", + "xls", + "user", + "has_start_time", + "shared", + "shared_data", + "deleted_at", + "deleted_by", + ) + + # pylint: disable=no-self-use def get_metadata(self, obj): + """ + Returns XForn ``obj`` metadata. + """ xform_metadata = [] if obj: - xform_metadata = cache.get( - '{}{}'.format(XFORM_METADATA_CACHE, obj.pk)) + xform_metadata = cache.get(f"{XFORM_METADATA_CACHE}{obj.pk}") if xform_metadata: return xform_metadata xform_metadata = list( MetaDataSerializer( - obj.metadata_set.all(), many=True, context=self.context) - .data) - cache.set('{}{}'.format(XFORM_METADATA_CACHE, obj.pk), - xform_metadata) + obj.metadata_set.all(), many=True, context=self.context + ).data + ) + cache.set(f"{XFORM_METADATA_CACHE}{obj.pk}", xform_metadata) return xform_metadata @@ -402,11 +502,11 @@ def validate_public_key(self, value): # pylint: disable=no-self-use package """ try: - load_pem_public_key( - value.encode('utf-8'), backend=default_backend()) - except ValueError: + load_pem_public_key(value.encode("utf-8"), backend=default_backend()) + except ValueError as e: raise serializers.ValidationError( - _('The public key is not a valid base64 RSA key')) + _("The public key is not a valid base64 RSA key") + ) from e return clean_public_key(value) def _check_if_allowed_public(self, value): # pylint: disable=no-self-use @@ -415,8 +515,7 @@ def _check_if_allowed_public(self, value): # pylint: disable=no-self-use forms """ if not settings.ALLOW_PUBLIC_DATASETS and value: - raise serializers.ValidationError( - _('Public forms are currently disabled.')) + raise serializers.ValidationError(_("Public forms are currently disabled.")) return value def validate_public_data(self, value): @@ -431,126 +530,169 @@ def validate_public(self, value): """ return self._check_if_allowed_public(value) + # pylint: disable=no-self-use def get_form_versions(self, obj): + """ + Returns all form versions. + """ versions = [] if obj: - versions = cache.get('{}{}'.format(XFORM_DATA_VERSIONS, obj.pk)) + versions = cache.get(f"{XFORM_DATA_VERSIONS}{obj.pk}") if versions: return versions - elif obj.num_of_submissions > SUBMISSION_RETRIEVAL_THRESHOLD: + if obj.num_of_submissions > SUBMISSION_RETRIEVAL_THRESHOLD: return [] versions = list( Instance.objects.filter(xform=obj, deleted_at__isnull=True) - .values('version').annotate(total=Count('version'))) + .values("version") + .annotate(total=Count("version")) + ) if versions: - cache.set('{}{}'.format(XFORM_DATA_VERSIONS, obj.pk), - list(versions)) + cache.set(f"{XFORM_DATA_VERSIONS}{obj.pk}", list(versions)) return versions +# pylint: disable=abstract-method class XFormCreateSerializer(XFormSerializer): + """ + XForm serializer that is only relevant during the XForm publishing process. + """ + has_id_string_changed = serializers.SerializerMethodField() + # pylint: disable=no-self-use def get_has_id_string_changed(self, obj): + """ + Returns the value of ``obj.has_id_string_changed`` + """ return obj.has_id_string_changed +# pylint: disable=abstract-method class XFormListSerializer(serializers.Serializer): - formID = serializers.ReadOnlyField(source='id_string') - name = serializers.ReadOnlyField(source='title') + """ + XForm serializer for OpenRosa form list API. + """ + + formID = serializers.ReadOnlyField(source="id_string") # noqa + name = serializers.ReadOnlyField(source="title") version = serializers.ReadOnlyField() hash = serializers.ReadOnlyField() - descriptionText = serializers.ReadOnlyField(source='description') - downloadUrl = serializers.SerializerMethodField('get_url') - manifestUrl = serializers.SerializerMethodField('get_manifest_url') + descriptionText = serializers.ReadOnlyField(source="description") # noqa + downloadUrl = serializers.SerializerMethodField("get_url") # noqa + manifestUrl = serializers.SerializerMethodField("get_manifest_url") # noqa @check_obj def get_url(self, obj): - kwargs = {'pk': obj.pk, 'username': obj.user.username} - request = self.context.get('request') + """ + Returns XForm download URL. + """ + kwargs = {"pk": obj.pk, "username": obj.user.username} + request = self.context.get("request") - return reverse('download_xform', kwargs=kwargs, request=request) + return reverse("download_xform", kwargs=kwargs, request=request) @check_obj def get_manifest_url(self, obj): - kwargs = {'pk': obj.pk, 'username': obj.user.username} - request = self.context.get('request') - object_list = MetaData.objects.filter(data_type='media', - object_id=obj.pk) + """ + Return manifest URL. + """ + kwargs = {"pk": obj.pk, "username": obj.user.username} + request = self.context.get("request") + object_list = MetaData.objects.filter(data_type="media", object_id=obj.pk) if object_list: - return reverse('manifest-url', kwargs=kwargs, request=request) + return reverse("manifest-url", kwargs=kwargs, request=request) return None class XFormManifestSerializer(serializers.Serializer): + """XForm Manifest serializer class.""" + filename = serializers.SerializerMethodField() hash = serializers.SerializerMethodField() - downloadUrl = serializers.SerializerMethodField('get_url') + downloadUrl = serializers.SerializerMethodField("get_url") # noqa @check_obj def get_url(self, obj): + """ + Return media download URL. + """ kwargs = { - 'pk': obj.content_object.pk, - 'username': obj.content_object.user.username, - 'metadata': obj.pk + "pk": obj.content_object.pk, + "username": obj.content_object.user.username, + "metadata": obj.pk, } - request = self.context.get('request') + request = self.context.get("request") try: - fmt = obj.data_value[obj.data_value.rindex('.') + 1:] + fmt_index = obj.data_value.rindex(".") + 1 + fmt = obj.data_value[fmt_index:] except ValueError: - fmt = 'csv' - url = reverse( - 'xform-media', kwargs=kwargs, request=request, format=fmt.lower()) + fmt = "csv" + url = reverse("xform-media", kwargs=kwargs, request=request, format=fmt.lower()) group_delimiter = self.context.get(GROUP_DELIMETER_TAG) repeat_index_tags = self.context.get(REPEAT_INDEX_TAGS) - if group_delimiter and repeat_index_tags and fmt == 'csv': - return (url+"?%s=%s&%s=%s" % ( - GROUP_DELIMETER_TAG, group_delimiter, REPEAT_INDEX_TAGS, - repeat_index_tags)) + if group_delimiter and repeat_index_tags and fmt == "csv": + return url + ( + f"?{GROUP_DELIMETER_TAG}={group_delimiter}" + f"&{REPEAT_INDEX_TAGS}={repeat_index_tags}" + ) return url + # pylint: disable=no-self-use @check_obj def get_hash(self, obj): + """ + Returns MD5 hash based on last_submission_time for a media linked form. + """ filename = obj.data_value hsh = obj.file_hash - parts = filename.split(' ') + parts = filename.split(" ") # filtered dataset is of the form "xform PK name", xform pk is the # second item # other file uploads other than linked datasets have a data_file - if len(parts) > 2 and obj.data_file == '': + if len(parts) > 2 and obj.data_file == "": dataset_type = parts[0] pk = parts[1] xform = None - if dataset_type == 'xform': - xform = XForm.objects.filter(pk=pk)\ - .only('last_submission_time').first() + if dataset_type == "xform": + xform = XForm.objects.filter(pk=pk).only("last_submission_time").first() else: - data_view = DataView.objects.filter(pk=pk)\ - .only('xform__last_submission_time').first() + data_view = ( + DataView.objects.filter(pk=pk) + .only("xform__last_submission_time") + .first() + ) if data_view: xform = data_view.xform if xform and xform.last_submission_time: - hsh = u'md5:%s' % (md5( - xform.last_submission_time.isoformat().encode( - 'utf-8')).hexdigest()) + md5_hash = hashlib.new( + "md5", + xform.last_submission_time.isoformat().encode("utf-8"), + usedforsecurity=False, + ).hexdigest() + hsh = f"md5:{md5_hash}" - return u"%s" % (hsh or 'md5:') + return f"{hsh or 'md5:'}" + # pylint: disable=no-self-use @check_obj def get_filename(self, obj): + """ + Returns media filename. + """ filename = obj.data_value - parts = filename.split(' ') + parts = filename.split(" ") # filtered dataset is of the form "xform PK name", filename is the # third item if len(parts) > 2: - filename = u'%s.csv' % parts[2] + filename = f"{parts[2]}.csv" else: try: URLValidator()(filename) @@ -563,27 +705,34 @@ def get_filename(self, obj): return filename +# pylint: disable=too-few-public-methods class XFormVersionListSerializer(serializers.ModelSerializer): + """ + XFormVersion list API serializer + """ + xform = serializers.HyperlinkedRelatedField( - view_name='xform-detail', lookup_field='pk', - queryset=XForm.objects.filter(deleted_at__isnull=True) + view_name="xform-detail", + lookup_field="pk", + queryset=XForm.objects.filter(deleted_at__isnull=True), ) url = MultiLookupIdentityField( - view_name='form-version-detail', - lookup_fields=(('xform__pk', 'pk'), ('version', 'version_id')) + view_name="form-version-detail", + lookup_fields=(("xform__pk", "pk"), ("version", "version_id")), ) xml = MultiLookupIdentityField( - view_name='form-version-detail', - format='xml', - lookup_fields=(('xform__pk', 'pk'), ('version', 'version_id')) + view_name="form-version-detail", + format="xml", + lookup_fields=(("xform__pk", "pk"), ("version", "version_id")), ) created_by = serializers.HyperlinkedRelatedField( - view_name='user-detail', - lookup_field='username', + view_name="user-detail", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME) + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), ) class Meta: model = XFormVersion - exclude = ('json', 'xls', 'id') + exclude = ("json", "xls", "id") diff --git a/onadata/libs/templates/rest_framework_swagger/index.html b/onadata/libs/templates/rest_framework_swagger/index.html index c11679bdc0..a12911d154 100644 --- a/onadata/libs/templates/rest_framework_swagger/index.html +++ b/onadata/libs/templates/rest_framework_swagger/index.html @@ -1,4 +1,4 @@ -{% load staticfiles %} +{% load static %} diff --git a/onadata/libs/tests/filters/test_instance_permission_filter.py b/onadata/libs/tests/filters/test_instance_permission_filter.py index 81203bae54..79ddbcf541 100644 --- a/onadata/libs/tests/filters/test_instance_permission_filter.py +++ b/onadata/libs/tests/filters/test_instance_permission_filter.py @@ -79,7 +79,7 @@ def test_metadata_filter_for_user_with_xform_perms(self): request = self.factory.get('/', data=params, **self.extra) response = self.view(request) - self.assertEquals(len(response.data), 1) + self.assertEqual(len(response.data), 1) def test_metadata_filter_for_user_without_xform_perms(self): self._create_user_and_login("alice", "password") @@ -91,7 +91,7 @@ def test_metadata_filter_for_user_without_xform_perms(self): request = self.factory.get('/', data=params, **extra) response = self.view(request) - self.assertEquals(len(response.data), 0) + self.assertEqual(len(response.data), 0) def test_filter_for_foreign_instance_request(self): self._create_user_and_login("alice", "password") @@ -110,7 +110,7 @@ def test_filter_for_foreign_instance_request(self): request = self.factory.get('/', data=params, **extra) response = self.view(request) - self.assertEquals(len(response.data), 0) + self.assertEqual(len(response.data), 0) def test_filter_for_dataview_metadata_instance_request(self): # """Dataview IDs should not yield any submission metadata""" @@ -128,7 +128,7 @@ def test_filter_for_dataview_metadata_instance_request(self): request = self.factory.get('/', data=params, **self.extra) response = self.view(request) - self.assertEquals(len(response.data), 0) + self.assertEqual(len(response.data), 0) def test_filter_given_user_without_permissions_to_xform(self): """Instance metadata isn't listed for users without form perms""" @@ -152,7 +152,7 @@ def test_filter_given_user_without_permissions_to_xform(self): request = self.factory.get('/', data=params, **self.extra) response = self.view(request) - self.assertEquals(len(response.data), 0) + self.assertEqual(len(response.data), 0) def test_filter_given_dataview_in_project_without_instance(self): """Meta data for submissions shouldn't be accessible from dataview""" @@ -173,4 +173,4 @@ def test_filter_given_dataview_in_project_without_instance(self): request = self.factory.get('/', data=params, **self.extra) response = self.view(request) - self.assertEquals(len(response.data), 0) + self.assertEqual(len(response.data), 0) diff --git a/onadata/libs/tests/serializers/fixtures/sample.geojson b/onadata/libs/tests/serializers/fixtures/sample.geojson new file mode 100644 index 0000000000..1ae1b9cfdb --- /dev/null +++ b/onadata/libs/tests/serializers/fixtures/sample.geojson @@ -0,0 +1,13 @@ +{ + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 125.6, + 10.1 + ] + }, + "properties": { + "name": "Dinagat Islands" + } +} \ No newline at end of file diff --git a/onadata/libs/tests/serializers/test_attachment_serializer.py b/onadata/libs/tests/serializers/test_attachment_serializer.py index ead4e8adb3..3c572bc009 100644 --- a/onadata/libs/tests/serializers/test_attachment_serializer.py +++ b/onadata/libs/tests/serializers/test_attachment_serializer.py @@ -59,9 +59,9 @@ def setUp(self): def test_get_field_xpath_of_an_object(self): path = get_path(self.data, self.question, []) - self.assertEquals(path, "group1/group2/photograph") + self.assertEqual(path, "group1/group2/photograph") # call 'get_path' twice to ensure the incorrect path bug has been # sorted; calling it the second time initially would return the # following path: "group1/group2/photograph/group1/group2/photograph" path = get_path(self.data, self.question, []) - self.assertEquals(path, "group1/group2/photograph") + self.assertEqual(path, "group1/group2/photograph") diff --git a/onadata/libs/tests/serializers/test_dataview_serializer.py b/onadata/libs/tests/serializers/test_dataview_serializer.py index af69656cbc..4f582ef6c0 100644 --- a/onadata/libs/tests/serializers/test_dataview_serializer.py +++ b/onadata/libs/tests/serializers/test_dataview_serializer.py @@ -73,7 +73,7 @@ def test_name_and_xform_are_unique(self): self.assertTrue(is_valid) serializer.save() - self.assertEquals(DataView.objects.count(), 1) + self.assertEqual(DataView.objects.count(), 1) # Try and save the same data again and confirm it fails serializer = DataViewSerializer( @@ -86,4 +86,4 @@ def test_name_and_xform_are_unique(self): serializer.errors.get( 'non_field_errors')[0].title() == expected_error_msg - self.assertEquals(DataView.objects.count(), 1) + self.assertEqual(DataView.objects.count(), 1) diff --git a/onadata/libs/tests/serializers/test_metadata_serializer.py b/onadata/libs/tests/serializers/test_metadata_serializer.py index 0190eacbea..7c92b5697f 100644 --- a/onadata/libs/tests/serializers/test_metadata_serializer.py +++ b/onadata/libs/tests/serializers/test_metadata_serializer.py @@ -3,11 +3,11 @@ Test onadata.libs.serializers.metadata_serializer """ import os + from django.core.files.uploadedfile import InMemoryUploadedFile from django.test.utils import override_settings -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.libs.serializers.metadata_serializer import MetaDataSerializer @@ -23,8 +23,7 @@ def test_data_value_is_required(self): data = {} serializer = MetaDataSerializer(data=data) self.assertFalse(serializer.is_valid()) - self.assertEqual(serializer.errors['data_value'], - [u'This field is required.']) + self.assertEqual(serializer.errors["data_value"], ["This field is required."]) def test_media_url_validation(self): """ @@ -33,16 +32,21 @@ def test_media_url_validation(self): self._login_user_and_profile() self._publish_form_with_hxl_support() data = { - 'data_value': 'http://example.com', - 'data_type': 'media', - 'xform': self.xform.pk + "data_value": "http://example.com", + "data_type": "media", + "xform": self.xform.pk, } serializer = MetaDataSerializer(data=data) self.assertFalse(serializer.is_valid()) self.assertEqual( - serializer.errors['data_value'], - [(u"Cannot get filename from URL %(data_value)s. URL should " - u"include the filename e.g %(data_value)s/data.csv" % data)]) + serializer.errors["data_value"], + [ + ( + "Cannot get filename from URL %(data_value)s. URL should " + "include the filename e.g %(data_value)s/data.csv" % data + ) + ], + ) @override_settings(SUPPORTED_MEDIA_UPLOAD_TYPES=[]) def test_unsupported_media_files(self): @@ -51,22 +55,24 @@ def test_unsupported_media_files(self): """ self._login_user_and_profile() self._publish_form_with_hxl_support() - data_value = 'sample.svg' - path = os.path.join(os.path.dirname(__file__), 'fixtures', - 'sample.svg') + data_value = "sample.svg" + path = os.path.join(os.path.dirname(__file__), "fixtures", "sample.svg") with open(path) as f: f = InMemoryUploadedFile( - f, 'media', data_value, 'application/octet-stream', 2324, None) + f, "media", data_value, "application/octet-stream", 2324, None + ) data = { - 'data_value': data_value, - 'data_file': f, - 'data_type': 'media', - 'xform': self.xform.pk + "data_value": data_value, + "data_file": f, + "data_type": "media", + "xform": self.xform.pk, } serializer = MetaDataSerializer(data=data) self.assertFalse(serializer.is_valid()) - self.assertEqual(serializer.errors['data_file'], - [("Unsupported media file type image/svg+xml")]) + self.assertEqual( + serializer.errors["data_file"], + [("Unsupported media file type image/svg+xml")], + ) def test_svg_media_files(self): """ @@ -74,12 +80,36 @@ def test_svg_media_files(self): """ self._login_user_and_profile() self._publish_form_with_hxl_support() - data_value = 'sample.svg' + data_value = "sample.svg" + path = os.path.join(os.path.dirname(__file__), "fixtures", "sample.svg") + with open(path) as f: + f = InMemoryUploadedFile( + f, "media", data_value, "application/octet-stream", 2324, None + ) + data = { + "data_value": data_value, + "data_file": f, + "data_type": "media", + "xform": self.xform.pk, + } + serializer = MetaDataSerializer(data=data) + self.assertTrue(serializer.is_valid()) + self.assertEqual( + serializer.validated_data["data_file_type"], "image/svg+xml" + ) + + def test_geojson_media_files(self): + """ + Test that an geojson file is uploaded ok + """ + self._login_user_and_profile() + self._publish_form_with_hxl_support() + data_value = 'sample.geojson' path = os.path.join(os.path.dirname(__file__), 'fixtures', - 'sample.svg') + 'sample.geojson') with open(path) as f: f = InMemoryUploadedFile( - f, 'media', data_value, 'application/octet-stream', 2324, None) + f, 'media', data_value, None, 2324, None) data = { 'data_value': data_value, 'data_file': f, @@ -88,5 +118,12 @@ def test_svg_media_files(self): } serializer = MetaDataSerializer(data=data) self.assertTrue(serializer.is_valid()) - self.assertEquals(serializer.validated_data['data_file_type'], - 'image/svg+xml') + self.assertEqual( + serializer.validated_data['data_file_type'], + 'application/geo+json') + self.assertEqual( + serializer.validated_data['data_value'], + 'sample.geojson') + self.assertEqual( + serializer.validated_data['data_type'], + 'media') diff --git a/onadata/libs/tests/test_authentication.py b/onadata/libs/tests/test_authentication.py index 2dc55a94a8..caba609e8d 100644 --- a/onadata/libs/tests/test_authentication.py +++ b/onadata/libs/tests/test_authentication.py @@ -103,8 +103,8 @@ def test_authenticates_if_token_is_valid(self): returned_user, returned_token, ) = self.temp_token_authentication.authenticate_credentials(token.key) - self.assertEquals(user, returned_user) - self.assertEquals(token, returned_token) + self.assertEqual(user, returned_user) + self.assertEqual(token, returned_token) class TestTempTokenURLParameterAuthentication(TestCase): @@ -154,7 +154,10 @@ def test_check_lockout(self): # Uses X_REAL_IP if present self.assertNotIn("HTTP_X_REAL_IP", request.META) - request.META.update({"HTTP_X_REAL_IP": "1.2.3.4"}) + extra = {"HTTP_X_REAL_IP": "1.2.3.4"} + extra.update(self.extra) + request = self.factory.get("/", **extra) + self.assertEqual(check_lockout(request), ("1.2.3.4", "bob")) def test_exception_on_username_with_whitespaces(self): diff --git a/onadata/libs/tests/test_renderers.py b/onadata/libs/tests/test_renderers.py index bc777a56bd..037c6d6aba 100644 --- a/onadata/libs/tests/test_renderers.py +++ b/onadata/libs/tests/test_renderers.py @@ -30,7 +30,7 @@ def test_floip_rows_list(self): ['2018-03-05T13:57:26+00:00', 19, None, 1, 'age', 10, None] ] # yapf: disable result = [_ for _ in floip_rows_list(data)] - self.assertEquals(result, expected_data) + self.assertEqual(result, expected_data) def test_floip_rows_list_w_flow_fields(self): # pylint: disable=C0103 """ @@ -50,4 +50,4 @@ def test_floip_rows_list_w_flow_fields(self): # pylint: disable=C0103 ['2018-03-05T13:57:26+00:00', 34, '789', '345', 'age', 10, None] ] # yapf: disable result = [_ for _ in floip_rows_list(data)] - self.assertEquals(result, expected_data) + self.assertEqual(result, expected_data) diff --git a/onadata/libs/tests/utils/test_api_export_tools.py b/onadata/libs/tests/utils/test_api_export_tools.py index 9f1bb9736c..2fd263e523 100644 --- a/onadata/libs/tests/utils/test_api_export_tools.py +++ b/onadata/libs/tests/utils/test_api_export_tools.py @@ -76,7 +76,7 @@ def test_process_async_export_returns_existing_export(self): resp = process_async_export( request, self.xform, export_type, options=options) - self.assertEquals(resp['job_status'], status_msg[SUCCESSFUL]) + self.assertEqual(resp['job_status'], status_msg[SUCCESSFUL]) self.assertIn("export_url", resp) # pylint: disable=invalid-name diff --git a/onadata/libs/tests/utils/test_chart_tools.py b/onadata/libs/tests/utils/test_chart_tools.py index c305cacf8f..dd76714d76 100644 --- a/onadata/libs/tests/utils/test_chart_tools.py +++ b/onadata/libs/tests/utils/test_chart_tools.py @@ -85,7 +85,7 @@ def test_build_chart_data_for_fields_with_accents(self): count = XForm.objects.count() self._publish_xls_file(xls_path) - self.assertEquals(XForm.objects.count(), count + 1) + self.assertEqual(XForm.objects.count(), count + 1) xform = XForm.objects.get(id_string='sample_accent') self.assertEqual(xform.title, "sample_accent") @@ -113,7 +113,7 @@ def test_build_chart_data_for_fields_with_apostrophies(self): count = XForm.objects.count() self._publish_xls_file(xls_path) - self.assertEquals(XForm.objects.count(), count + 1) + self.assertEqual(XForm.objects.count(), count + 1) xform = XForm.objects.get(id_string='sample_accent') self.assertEqual(xform.title, "sample_accent") diff --git a/onadata/libs/tests/utils/test_csv_builder.py b/onadata/libs/tests/utils/test_csv_builder.py index e87ffcd233..095c7bdcb3 100644 --- a/onadata/libs/tests/utils/test_csv_builder.py +++ b/onadata/libs/tests/utils/test_csv_builder.py @@ -1220,7 +1220,7 @@ def test_export_data_for_xforms_without_submissions(self): self._publish_xls_fixture_set_xform(fixture) # Confirm form has not submissions so far - self.assertEquals(self.xform.instances.count(), 0) + self.assertEqual(self.xform.instances.count(), 0) # Generate csv export for form csv_df_builder = CSVDataFrameBuilder( self.user.username, self.xform.id_string, include_images=False) @@ -1253,7 +1253,7 @@ def test_export_data_for_xforms_with_newer_submissions(self): self._publish_xls_fixture_set_xform(fixture) # Confirm form has not submissions so far - self.assertEquals(self.xform.instances.count(), 0) + self.assertEqual(self.xform.instances.count(), 0) # Generate csv export for form csv_df_builder = CSVDataFrameBuilder( self.user.username, self.xform.id_string, include_images=False) @@ -1320,7 +1320,7 @@ def test_export_raises_NoRecordsFound_for_form_without_instances(self): self._publish_xls_fixture_set_xform(fixture) # Confirm form has not submissions so far - self.assertEquals(self.xform.instances.count(), 0) + self.assertEqual(self.xform.instances.count(), 0) # Generate csv export for form csv_df_builder_1 = CSVDataFrameBuilder( self.user.username, diff --git a/onadata/libs/tests/utils/test_csv_import.py b/onadata/libs/tests/utils/test_csv_import.py index 0991a6eea5..fb55f8ce39 100644 --- a/onadata/libs/tests/utils/test_csv_import.py +++ b/onadata/libs/tests/utils/test_csv_import.py @@ -1,6 +1,5 @@ from __future__ import unicode_literals -import json import os import re from builtins import open @@ -16,8 +15,11 @@ from onadata.apps.logger.models import Instance, XForm from onadata.apps.main.tests.test_base import TestBase from onadata.apps.main.models import MetaData -from onadata.apps.messaging.constants import \ - XFORM, SUBMISSION_EDITED, SUBMISSION_CREATED +from onadata.apps.messaging.constants import ( + XFORM, + SUBMISSION_EDITED, + SUBMISSION_CREATED, +) from onadata.libs.utils import csv_import from onadata.libs.utils.common_tags import IMPORTED_VIA_CSV_BY from onadata.libs.utils.csv_import import get_submission_meta_dict @@ -26,143 +28,149 @@ def strip_xml_uuid(s): - return re.sub(b'\S*uuid\S*', b'', s.rstrip(b'\n')) # noqa + return re.sub(b"\S*uuid\S*", b"", s.rstrip(b"\n")) # noqa class CSVImportTestCase(TestBase): def setUp(self): super(CSVImportTestCase, self).setUp() - self.fixtures_dir = os.path.join(settings.PROJECT_ROOT, 'libs', - 'tests', 'utils', 'fixtures') - self.good_csv = open(os.path.join(self.fixtures_dir, 'good.csv'), 'rb') - self.bad_csv = open(os.path.join(self.fixtures_dir, 'bad.csv'), 'rb') - self.xls_file_path = os.path.join(self.fixtures_dir, 'tutorial.xlsx') - self.good_xls = open( - os.path.join(self.fixtures_dir, 'good.xlsx'), 'rb') + self.fixtures_dir = os.path.join( + settings.PROJECT_ROOT, "libs", "tests", "utils", "fixtures" + ) + self.good_csv = open(os.path.join(self.fixtures_dir, "good.csv"), "rb") + self.bad_csv = open(os.path.join(self.fixtures_dir, "bad.csv"), "rb") + self.xls_file_path = os.path.join(self.fixtures_dir, "tutorial.xlsx") + self.good_xls = open(os.path.join(self.fixtures_dir, "good.xlsx"), "rb") def test_get_submission_meta_dict(self): self._publish_xls_file(self.xls_file_path) xform = XForm.objects.get() meta = get_submission_meta_dict(xform, None) self.assertEqual(len(meta), 2) - self.assertTrue('instanceID' in meta[0]) + self.assertTrue("instanceID" in meta[0]) self.assertEqual(meta[1], 0) - instance_id = 'uuid:9118a3fc-ab99-44cf-9a97-1bb1482d8e2b' + instance_id = "uuid:9118a3fc-ab99-44cf-9a97-1bb1482d8e2b" meta = get_submission_meta_dict(xform, instance_id) - self.assertTrue('instanceID' in meta[0]) - self.assertEqual(meta[0]['instanceID'], instance_id) + self.assertTrue("instanceID" in meta[0]) + self.assertEqual(meta[0]["instanceID"], instance_id) self.assertEqual(meta[1], 0) def test_submit_csv_param_sanity_check(self): - resp = csv_import.submit_csv('userX', XForm(), 123456) - self.assertIsNotNone(resp.get('error')) + resp = csv_import.submit_csv("userX", XForm(), 123456) + self.assertIsNotNone(resp.get("error")) - @mock.patch('onadata.libs.utils.csv_import.safe_create_instance') + @mock.patch("onadata.libs.utils.csv_import.safe_create_instance") def test_submit_csv_xml_params(self, safe_create_instance): self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() safe_create_instance.return_value = {} - single_csv = open(os.path.join(self.fixtures_dir, 'single.csv'), 'rb') + single_csv = open(os.path.join(self.fixtures_dir, "single.csv"), "rb") csv_import.submit_csv(self.user.username, self.xform, single_csv) xml_file_param = BytesIO( - open(os.path.join(self.fixtures_dir, 'single.xml'), 'rb').read()) + open(os.path.join(self.fixtures_dir, "single.xml"), "rb").read() + ) safe_create_args = list(safe_create_instance.call_args[0]) - self.assertEqual(safe_create_args[0], self.user.username, - 'Wrong username passed') + self.assertEqual( + safe_create_args[0], self.user.username, "Wrong username passed" + ) self.assertEqual( strip_xml_uuid(safe_create_args[1].getvalue()), strip_xml_uuid(xml_file_param.getvalue()), - 'Wrong xml param passed') - self.assertEqual(safe_create_args[2], [], - 'Wrong media array param passed') - self.assertEqual(safe_create_args[3], self.xform.uuid, - 'Wrong xform uuid passed') + "Wrong xml param passed", + ) + self.assertEqual(safe_create_args[2], [], "Wrong media array param passed") + self.assertEqual( + safe_create_args[3], self.xform.uuid, "Wrong xform uuid passed" + ) self.assertEqual(safe_create_args[4], None) - @mock.patch('onadata.libs.utils.csv_import.safe_create_instance') - @mock.patch('onadata.libs.utils.csv_import.dict2xmlsubmission') - def test_submit_csv_xml_location_property_test(self, d2x, - safe_create_instance): + @mock.patch("onadata.libs.utils.csv_import.safe_create_instance") + @mock.patch("onadata.libs.utils.csv_import.dict2xmlsubmission") + def test_submit_csv_xml_location_property_test(self, d2x, safe_create_instance): self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() safe_create_instance.return_value = [ None, ] - single_csv = open(os.path.join(self.fixtures_dir, 'single.csv'), 'rb') + single_csv = open(os.path.join(self.fixtures_dir, "single.csv"), "rb") csv_import.submit_csv(self.user.username, self.xform, single_csv) - test_location_val = '83.3595 -32.8601 0 1' - test_location2_val = '21.22474 -10.5601 50000 200' + test_location_val = "83.3595 -32.8601 0 1" + test_location2_val = "21.22474 -10.5601 50000 200" - self.assertNotEqual(d2x.call_args, None, - 'dict2xmlsubmission not called') + self.assertNotEqual(d2x.call_args, None, "dict2xmlsubmission not called") call_dict = d2x.call_args[0][0] self.assertEqual( - call_dict.get('test_location'), test_location_val, - 'Location prop test fail') + call_dict.get("test_location"), test_location_val, "Location prop test fail" + ) self.assertEqual( - call_dict.get('test_location2'), test_location2_val, - 'Location2 prop test fail') + call_dict.get("test_location2"), + test_location2_val, + "Location2 prop test fail", + ) def test_submit_csv_and_rollback(self): - xls_file_path = os.path.join(settings.PROJECT_ROOT, "apps", "main", - "tests", "fixtures", "tutorial.xlsx") + xls_file_path = os.path.join( + settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", "tutorial.xlsx" + ) self._publish_xls_file(xls_file_path) self.xform = XForm.objects.get() count = Instance.objects.count() csv_import.submit_csv(self.user.username, self.xform, self.good_csv) - self.assertEqual(Instance.objects.count(), count + 9, - 'submit_csv test Failed!') + self.assertEqual(Instance.objects.count(), count + 9, "submit_csv test Failed!") # Check that correct # of submissions belong to our user self.assertEqual( Instance.objects.filter(user=self.user).count(), count + 8, - 'submit_csv username check failed!') + "submit_csv username check failed!", + ) self.xform.refresh_from_db() self.assertEqual(self.xform.num_of_submissions, count + 9) # Test rollback on error and user feedback with patch( - 'onadata.libs.utils.csv_import.safe_create_instance' - ) as safe_create_mock: + "onadata.libs.utils.csv_import.safe_create_instance" + ) as safe_create_mock: safe_create_mock.side_effect = [AttributeError] initial_count = self.xform.num_of_submissions - resp = csv_import.submit_csv( - self.user.username, self.xform, self.good_csv) - self.assertEqual(resp, {'job_status': 'FAILURE'}) + resp = csv_import.submit_csv(self.user.username, self.xform, self.good_csv) + self.assertEqual(resp, {"job_status": "FAILURE"}) self.xform.refresh_from_db() - self.assertEqual( - initial_count, self.xform.num_of_submissions) + self.assertEqual(initial_count, self.xform.num_of_submissions) - @patch('onadata.libs.utils.logger_tools.send_message') + @patch("onadata.libs.utils.logger_tools.send_message") def test_submit_csv_edits(self, send_message_mock): - xls_file_path = os.path.join(settings.PROJECT_ROOT, "apps", "main", - "tests", "fixtures", "tutorial.xlsx") + xls_file_path = os.path.join( + settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", "tutorial.xlsx" + ) self._publish_xls_file(xls_file_path) self.xform = XForm.objects.get() csv_import.submit_csv(self.user.username, self.xform, self.good_csv) - self.assertEqual(Instance.objects.count(), 9, - 'submit_csv edits #1 test Failed!') + self.assertEqual( + Instance.objects.count(), 9, "submit_csv edits #1 test Failed!" + ) - edit_csv = open(os.path.join(self.fixtures_dir, 'edit.csv')) + edit_csv = open(os.path.join(self.fixtures_dir, "edit.csv")) edit_csv_str = edit_csv.read() edit_csv = BytesIO( edit_csv_str.format( - * [x.get('uuid') for x in Instance.objects.values('uuid')]) - .encode('utf-8')) + *[x.get("uuid") for x in Instance.objects.values("uuid")] + ).encode("utf-8") + ) count = Instance.objects.count() csv_import.submit_csv(self.user.username, self.xform, edit_csv) - self.assertEqual(Instance.objects.count(), count, - 'submit_csv edits #2 test Failed!') + self.assertEqual( + Instance.objects.count(), count, "submit_csv edits #2 test Failed!" + ) # message sent upon submission edit self.assertTrue(send_message_mock.called) send_message_mock.called_with(self.xform.id, XFORM, SUBMISSION_EDITED) @@ -173,98 +181,102 @@ def test_import_non_utf8_csv(self): self.xform = XForm.objects.get() count = Instance.objects.count() - non_utf8_csv = open(os.path.join(self.fixtures_dir, 'non_utf8.csv'), - 'rb') - result = csv_import.submit_csv(self.user.username, self.xform, - non_utf8_csv) + non_utf8_csv = open(os.path.join(self.fixtures_dir, "non_utf8.csv"), "rb") + result = csv_import.submit_csv(self.user.username, self.xform, non_utf8_csv) + self.assertEqual( + result.get("error"), + "CSV file must be utf-8 encoded", + "Incorrect error message returned.", + ) self.assertEqual( - result.get('error'), 'CSV file must be utf-8 encoded', - 'Incorrect error message returned.') - self.assertEqual(Instance.objects.count(), count, - 'Non unicode csv import rollback failed!') + Instance.objects.count(), count, "Non unicode csv import rollback failed!" + ) def test_reject_spaces_in_headers(self): - xls_file_path = os.path.join(self.fixtures_dir, 'space_in_header.xlsx') + xls_file_path = os.path.join(self.fixtures_dir, "space_in_header.xlsx") self._publish_xls_file(xls_file_path) self.xform = XForm.objects.get() - non_utf8csv = open(os.path.join(self.fixtures_dir, 'header_space.csv'), - 'rb') - result = csv_import.submit_csv(self.user.username, self.xform, - non_utf8csv) + non_utf8csv = open(os.path.join(self.fixtures_dir, "header_space.csv"), "rb") + result = csv_import.submit_csv(self.user.username, self.xform, non_utf8csv) self.assertEqual( - result.get('error'), - 'CSV file fieldnames should not contain spaces', - 'Incorrect error message returned.') + result.get("error"), + "CSV file fieldnames should not contain spaces", + "Incorrect error message returned.", + ) def test_nested_geo_paths_csv(self): - self.xls_file_path = os.path.join(self.fixtures_dir, - 'tutorial-nested-geo.xlsx') + self.xls_file_path = os.path.join(self.fixtures_dir, "tutorial-nested-geo.xlsx") self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() - good_csv = open(os.path.join(self.fixtures_dir, 'another_good.csv'), - 'rb') + good_csv = open(os.path.join(self.fixtures_dir, "another_good.csv"), "rb") csv_import.submit_csv(self.user.username, self.xform, good_csv) - self.assertEqual(Instance.objects.count(), 9, - 'submit_csv edits #1 test Failed!') + self.assertEqual( + Instance.objects.count(), 9, "submit_csv edits #1 test Failed!" + ) def test_csv_with_multiple_select_in_one_column(self): - self.xls_file_path = os.path.join(self.fixtures_dir, - 'form_with_multiple_select.xlsx') + self.xls_file_path = os.path.join( + self.fixtures_dir, "form_with_multiple_select.xlsx" + ) self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() good_csv = open( - os.path.join(self.fixtures_dir, - 'csv_import_with_multiple_select.csv'), - 'rb') + os.path.join(self.fixtures_dir, "csv_import_with_multiple_select.csv"), "rb" + ) csv_import.submit_csv(self.user.username, self.xform, good_csv) - self.assertEqual(Instance.objects.count(), 1, - 'submit_csv edits #1 test Failed!') + self.assertEqual( + Instance.objects.count(), 1, "submit_csv edits #1 test Failed!" + ) def test_csv_imports_are_tracked(self): """ Test that submissions created via CSV Import are tracked """ - self.xls_file_path = os.path.join(self.fixtures_dir, - 'form_with_multiple_select.xlsx') + self.xls_file_path = os.path.join( + self.fixtures_dir, "form_with_multiple_select.xlsx" + ) self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() good_csv = open( - os.path.join(self.fixtures_dir, - 'csv_import_with_multiple_select.csv'), - 'rb') + os.path.join(self.fixtures_dir, "csv_import_with_multiple_select.csv"), "rb" + ) csv_import.submit_csv(self.user.username, self.xform, good_csv) self.assertEqual(Instance.objects.count(), 1) - self.assertEqual(Instance.objects.first().status, 'imported_via_csv') + self.assertEqual(Instance.objects.first().status, "imported_via_csv") def test_csv_imports_initiator_stored(self): """ Test that the user who imported data via CSV is tracked """ - xls_file_path = os.path.join(settings.PROJECT_ROOT, "apps", "main", - "tests", "fixtures", "tutorial.xlsx") + xls_file_path = os.path.join( + settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", "tutorial.xlsx" + ) self._publish_xls_file(xls_file_path) self.xform = XForm.objects.get() csv_import.submit_csv(self.user.username, self.xform, self.good_csv) - self.assertEqual(Instance.objects.count(), 9, - 'submit_csv edits #1 test Failed!') + self.assertEqual( + Instance.objects.count(), 9, "submit_csv edits #1 test Failed!" + ) # Ensure bob user is tagged as the person who initiated the CSV Import metadata_qs = MetaData.objects.filter(data_type=IMPORTED_VIA_CSV_BY) self.assertEqual(metadata_qs.count(), 9) self.assertEqual(metadata_qs.first().data_value, self.user.username) def test_csv_with_repeats_import(self): - self.xls_file_path = os.path.join(self.this_directory, 'fixtures', - 'csv_export', - 'tutorial_w_repeats.xlsx') + self.xls_file_path = os.path.join( + self.this_directory, "fixtures", "csv_export", "tutorial_w_repeats.xlsx" + ) repeats_csv = open( - os.path.join(self.this_directory, 'fixtures', 'csv_export', - 'tutorial_w_repeats.csv'), - 'rb') + os.path.join( + self.this_directory, "fixtures", "csv_export", "tutorial_w_repeats.csv" + ), + "rb", + ) self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() pre_count = self.xform.instances.count() @@ -273,13 +285,18 @@ def test_csv_with_repeats_import(self): self.assertEqual(count, 1 + pre_count) def test_csv_with__more_than_4_repeats_import(self): - self.xls_file_path = os.path.join(self.this_directory, 'fixtures', - 'csv_export', - 'tutorial_w_repeats.xlsx') + self.xls_file_path = os.path.join( + self.this_directory, "fixtures", "csv_export", "tutorial_w_repeats.xlsx" + ) repeats_csv = open( - os.path.join(self.this_directory, 'fixtures', 'csv_export', - 'tutorial_w_repeats_import.csv'), - 'rb') + os.path.join( + self.this_directory, + "fixtures", + "csv_export", + "tutorial_w_repeats_import.csv", + ), + "rb", + ) self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() pre_count = self.xform.instances.count() @@ -290,14 +307,12 @@ def test_csv_with__more_than_4_repeats_import(self): instance = self.xform.instances.last() # repeats should be 6 - self.assertEqual(6, len(instance.json.get('children'))) + self.assertEqual(6, len(instance.json.get("children"))) - @mock.patch('onadata.libs.utils.csv_import.AsyncResult') + @mock.patch("onadata.libs.utils.csv_import.AsyncResult") def test_get_async_csv_submission_status(self, AsyncResult): result = csv_import.get_async_csv_submission_status(None) - self.assertEqual(result, - {'error': 'Empty job uuid', - 'job_status': 'FAILURE'}) + self.assertEqual(result, {"error": "Empty job uuid", "job_status": "FAILURE"}) class BacklogLimitExceededMockAsyncResult(object): def __init__(self): @@ -308,100 +323,104 @@ def state(self): raise BacklogLimitExceeded() AsyncResult.return_value = BacklogLimitExceededMockAsyncResult() - result = csv_import.get_async_csv_submission_status('x-y-z') - self.assertEqual(result, {'job_status': 'PENDING'}) + result = csv_import.get_async_csv_submission_status("x-y-z") + self.assertEqual(result, {"job_status": "PENDING"}) class MockAsyncResult(object): def __init__(self): - self.result = self.state = 'SUCCESS' + self.result = self.state = "SUCCESS" def get(self): - return {'job_status': 'SUCCESS'} + return {"job_status": "SUCCESS"} AsyncResult.return_value = MockAsyncResult() - result = csv_import.get_async_csv_submission_status('x-y-z') - self.assertEqual(result, {'job_status': 'SUCCESS'}) + result = csv_import.get_async_csv_submission_status("x-y-z") + self.assertEqual(result, {"job_status": "SUCCESS"}) class MockAsyncResult2(object): def __init__(self): - self.result = self.state = 'PROGRESS' + self.result = self.state = "PROGRESS" self.info = { "info": [], "job_status": "PROGRESS", "progress": 4000, - "total": 70605 + "total": 70605, } AsyncResult.return_value = MockAsyncResult2() - result = csv_import.get_async_csv_submission_status('x-y-z') - self.assertEqual(result, {'info': [], 'job_status': 'PROGRESS', - 'progress': 4000, 'total': 70605}) + result = csv_import.get_async_csv_submission_status("x-y-z") + self.assertEqual( + result, + {"info": [], "job_status": "PROGRESS", "progress": 4000, "total": 70605}, + ) class MockAsyncResultIOError(object): def __init__(self): self.result = IOError("File not found!") - self.state = 'FAILURE' + self.state = "FAILURE" AsyncResult.return_value = MockAsyncResultIOError() - result = csv_import.get_async_csv_submission_status('x-y-z') - self.assertEqual(result, - {'error': 'File not found!', - 'job_status': 'FAILURE'}) + result = csv_import.get_async_csv_submission_status("x-y-z") + self.assertEqual(result, {"error": "File not found!", "job_status": "FAILURE"}) # shouldn't fail if info is not of type dict class MockAsyncResult2(object): def __init__(self): - self.result = self.state = 'PROGRESS' + self.result = self.state = "PROGRESS" self.info = None AsyncResult.return_value = MockAsyncResult2() - result = csv_import.get_async_csv_submission_status('x-y-z') - self.assertEqual(result, {'job_status': 'PROGRESS'}) + result = csv_import.get_async_csv_submission_status("x-y-z") + self.assertEqual(result, {"job_status": "PROGRESS"}) def test_submission_xls_to_csv(self): """Test that submission_xls_to_csv converts to csv""" - c_csv_file = csv_import.submission_xls_to_csv( - self.good_xls) + c_csv_file = csv_import.submission_xls_to_csv(self.good_xls) c_csv_file.seek(0) - c_csv_reader = ucsv.DictReader(c_csv_file, encoding='utf-8-sig') - g_csv_reader = ucsv.DictReader(self.good_csv, encoding='utf-8-sig') + c_csv_reader = ucsv.DictReader(c_csv_file, encoding="utf-8-sig") + g_csv_reader = ucsv.DictReader(self.good_csv, encoding="utf-8-sig") - self.assertEqual( - g_csv_reader.fieldnames[10], c_csv_reader.fieldnames[10]) + self.assertEqual(g_csv_reader.fieldnames[10], c_csv_reader.fieldnames[10]) - @mock.patch('onadata.libs.utils.csv_import.safe_create_instance') + @mock.patch("onadata.libs.utils.csv_import.safe_create_instance") def test_submit_csv_instance_id_consistency(self, safe_create_instance): self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() safe_create_instance.return_value = {} - single_csv = open(os.path.join(self.fixtures_dir, 'single.csv'), 'rb') + single_csv = open(os.path.join(self.fixtures_dir, "single.csv"), "rb") csv_import.submit_csv(self.user.username, self.xform, single_csv) xml_file_param = BytesIO( - open(os.path.join(self.fixtures_dir, 'single.xml'), 'rb').read()) + open(os.path.join(self.fixtures_dir, "single.xml"), "rb").read() + ) safe_create_args = list(safe_create_instance.call_args[0]) instance_xml = fromstring(safe_create_args[1].getvalue()) single_instance_xml = fromstring(xml_file_param.getvalue()) - instance_id = [ - m.find('instanceID').text for m in instance_xml.findall('meta')][0] - single_instance_id = [m.find('instanceID').text for m in - single_instance_xml.findall('meta')][0] + instance_id = [m.find("instanceID").text for m in instance_xml.findall("meta")][ + 0 + ] + single_instance_id = [ + m.find("instanceID").text for m in single_instance_xml.findall("meta") + ][0] self.assertEqual( - len(instance_id), len(single_instance_id), - "Same uuid length in generated xml") + len(instance_id), + len(single_instance_id), + "Same uuid length in generated xml", + ) - @patch('onadata.libs.utils.logger_tools.send_message') + @patch("onadata.libs.utils.logger_tools.send_message") def test_data_upload(self, send_message_mock): """Data upload for submissions with no uuids""" self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() count = Instance.objects.count() - single_csv = open(os.path.join( - self.fixtures_dir, 'single_data_upload.csv'), 'rb') + single_csv = open( + os.path.join(self.fixtures_dir, "single_data_upload.csv"), "rb" + ) csv_import.submit_csv(self.user.username, self.xform, single_csv) self.xform.refresh_from_db() self.assertEqual(self.xform.num_of_submissions, count + 1) @@ -428,44 +447,52 @@ def test_excel_date_conversion(self): """ self._create_user_and_login() - data = {'name': 'data'} + data = {"name": "data"} survey = self.md_to_pyxform_survey(date_md_form, kwargs=data) - survey['sms_keyword'] = survey['id_string'] + survey["sms_keyword"] = survey["id_string"] project = get_user_default_project(self.user) - xform = XForm(created_by=self.user, user=self.user, - xml=survey.to_xml(), json=survey.to_json(), - project=project) + xform = XForm( + created_by=self.user, + user=self.user, + xml=survey.to_xml(), + json=survey.to_json(), + project=project, + ) xform.save() - date_csv = open(os.path.join( - self.fixtures_dir, 'date.csv'), 'rb') + date_csv = open(os.path.join(self.fixtures_dir, "date.csv"), "rb") date_csv.seek(0) - csv_reader = ucsv.DictReader(date_csv, encoding='utf-8-sig') + csv_reader = ucsv.DictReader(date_csv, encoding="utf-8-sig") xl_dates = [] xl_datetime = [] # xl dates for row in csv_reader: - xl_dates.append(row.get('tdate')) - xl_datetime.append(row.get('now')) + xl_dates.append(row.get("tdate")) + xl_datetime.append(row.get("now")) csv_import.submit_csv(self.user.username, xform, date_csv) # converted dates - conv_dates = [instance.json.get('tdate') - for instance in Instance.objects.filter( - xform=xform).order_by('date_created')] - conv_datetime = [instance.json.get('now') - for instance in Instance.objects.filter( - xform=xform).order_by('date_created')] - - self.assertEqual(xl_dates, ['3/1/2019', '2/26/2019']) + conv_dates = [ + instance.json.get("tdate") + for instance in Instance.objects.filter(xform=xform).order_by( + "date_created" + ) + ] + conv_datetime = [ + instance.json.get("now") + for instance in Instance.objects.filter(xform=xform).order_by( + "date_created" + ) + ] + + self.assertEqual(xl_dates, ["3/1/2019", "2/26/2019"]) self.assertEqual( - xl_datetime, - [u'6/12/2020 13:20', u'2019-03-11T16:00:51.147+02:00']) + xl_datetime, ["6/12/2020 13:20", "2019-03-11T16:00:51.147+02:00"] + ) self.assertEqual( - conv_datetime, - [u'2020-06-12T13:20:00', - u'2019-03-11T16:00:51.147000+02:00']) - self.assertEqual(conv_dates, ['2019-03-01', '2019-02-26']) + conv_datetime, ["2020-06-12T13:20:00", "2019-03-11T16:00:51.147000+02:00"] + ) + self.assertEqual(conv_dates, ["2019-03-01", "2019-02-26"]) def test_enforces_data_type_and_rollback(self): """ @@ -473,17 +500,15 @@ def test_enforces_data_type_and_rollback(self): during the process are rolled back """ # Test integer constraint is enforced - xls_file_path = os.path.join(settings.PROJECT_ROOT, "apps", "main", - "tests", "fixtures", "tutorial.xlsx") + xls_file_path = os.path.join( + settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", "tutorial.xlsx" + ) self._publish_xls_file(xls_file_path) self.xform = XForm.objects.last() - bad_data = open( - os.path.join(self.fixtures_dir, 'bad_data.csv'), - 'rb') + bad_data = open(os.path.join(self.fixtures_dir, "bad_data.csv"), "rb") count = Instance.objects.count() - result = csv_import.submit_csv(self.user.username, self.xform, - bad_data) + result = csv_import.submit_csv(self.user.username, self.xform, bad_data) expected_error = ( "Invalid CSV data imported in row(s): {1: ['Unknown date format(s)" @@ -492,21 +517,23 @@ def test_enforces_data_type_and_rollback(self): "(s): 22.32'], 4: ['Unknown date format(s): 2014-0900'], " "5: ['Unknown date format(s): 2014-0901'], 6: ['Unknown " "date format(s): 2014-0902'], 7: ['Unknown date format(s):" - " 2014-0903']}") - self.assertEqual( - result.get('error'), - expected_error) + " 2014-0903']}" + ) + self.assertEqual(result.get("error"), expected_error) # Assert all created instances were rolled back self.assertEqual(count, Instance.objects.count()) def test_csv_import_with_overwrite(self): self._publish_xls_file(self.xls_file_path) - surveys = ['uuid1'] + surveys = ["uuid1"] - paths = [os.path.join( - self.fixtures_dir, 'tutorial', 'instances', s, 'submission.xml') - for s in surveys] + paths = [ + os.path.join( + self.fixtures_dir, "tutorial", "instances", s, "submission.xml" + ) + for s in surveys + ] for path in paths: self._make_submission(path) @@ -516,11 +543,11 @@ def test_csv_import_with_overwrite(self): self.assertEqual(count, 1) - single_csv = open(os.path.join(self.fixtures_dir, 'same_uuid.csv'), - 'rb') + single_csv = open(os.path.join(self.fixtures_dir, "same_uuid.csv"), "rb") - csv_import.submit_csv(self.user.username, self.xform, single_csv, - overwrite=True) + csv_import.submit_csv( + self.user.username, self.xform, single_csv, overwrite=True + ) self.xform.refresh_from_db() count = self.xform.instances.filter(deleted_at=None).count() @@ -536,21 +563,18 @@ def test_get_columns_by_type(self): ) self._publish_xls_file(self.xls_file_path) xform = XForm.objects.get() - columns = get_columns_by_type(["date"], json.loads(xform.json)) + columns = get_columns_by_type(["date"], xform.json_dict()) self.assertEqual( columns, ["section_A/date_of_survey", "section_B/year_established"] ) good_csv = open( - os.path.join( - self.fixtures_dir, "csv_import_with_multiple_select.csv" - ), "rb" + os.path.join(self.fixtures_dir, "csv_import_with_multiple_select.csv"), "rb" ) csv_import.submit_csv(self.user.username, xform, good_csv) self.assertEqual(Instance.objects.count(), 1) submission = Instance.objects.first() self.assertEqual(submission.status, "imported_via_csv") - self.assertEqual(submission.json["section_A/date_of_survey"], - "2015-09-10") + self.assertEqual(submission.json["section_A/date_of_survey"], "2015-09-10") self.assertTrue( submission.json["section_B/year_established"].startswith("1890") ) diff --git a/onadata/libs/tests/utils/test_email.py b/onadata/libs/tests/utils/test_email.py index d1f643c512..5236df217d 100644 --- a/onadata/libs/tests/utils/test_email.py +++ b/onadata/libs/tests/utils/test_email.py @@ -1,77 +1,83 @@ -from future.moves.urllib.parse import urlencode +from six.moves.urllib.parse import urlencode from django.test import RequestFactory from django.test.utils import override_settings from onadata.apps.main.tests.test_base import TestBase -from onadata.libs.utils.email import ( - get_verification_email_data, get_verification_url -) +from onadata.libs.utils.email import get_verification_email_data, get_verification_url VERIFICATION_URL = "http://ab.cd.ef" class TestEmail(TestBase): - def setUp(self): self.email = "john@doe.com" - self.username = "johndoe", + self.username = ("johndoe",) self.verification_key = "123abc" self.redirect_url = "http://red.ir.ect" - self.custom_request = RequestFactory().get( - '/path', data={'name': u'test'} - ) + self.custom_request = RequestFactory().get("/path", data={"name": "test"}) @override_settings(VERIFICATION_URL=None) def test_get_verification_url(self): # without redirect_url - verification_url = get_verification_url(**{ - "redirect_url": None, - "request": self.custom_request, - "verification_key": self.verification_key - }) + verification_url = get_verification_url( + **{ + "redirect_url": None, + "request": self.custom_request, + "verification_key": self.verification_key, + } + ) self.assertEqual( verification_url, - ('http://testserver/api/v1/profiles/verify_email?' - 'verification_key=%s' % self.verification_key), + ( + "http://testserver/api/v1/profiles/verify_email?" + "verification_key=%s" % self.verification_key + ), ) # with redirect_url - verification_url = get_verification_url(**{ - "redirect_url": self.redirect_url, - "request": self.custom_request, - "verification_key": self.verification_key - }) + verification_url = get_verification_url( + **{ + "redirect_url": self.redirect_url, + "request": self.custom_request, + "verification_key": self.verification_key, + } + ) - string_query_params = urlencode({ - 'verification_key': self.verification_key, - 'redirect_url': self.redirect_url - }) + string_query_params = urlencode( + { + "verification_key": self.verification_key, + "redirect_url": self.redirect_url, + } + ) self.assertEqual( verification_url, - ('http://testserver/api/v1/profiles/verify_email?%s' - % string_query_params) + ("http://testserver/api/v1/profiles/verify_email?%s" % string_query_params), ) def _get_email_data(self, include_redirect_url=False): - verification_url = get_verification_url(**{ - "redirect_url": include_redirect_url and self.redirect_url, - "request": self.custom_request, - "verification_key": self.verification_key - }) - - email_data = get_verification_email_data(**{ - "email": self.email, - "username": self.username, - "verification_url": verification_url, - "request": self.custom_request - }) - - self.assertIn('email', email_data) - self.assertIn(self.email, email_data.get('email')) - self.assertIn('subject', email_data) - self.assertIn('message_txt', email_data) + verification_url = get_verification_url( + **{ + "redirect_url": include_redirect_url and self.redirect_url, + "request": self.custom_request, + "verification_key": self.verification_key, + } + ) + + email_data = get_verification_email_data( + **{ + "email": self.email, + "username": self.username, + "verification_url": verification_url, + "request": self.custom_request, + } + ) + + self.assertIn("email", email_data) + self.assertIn(self.email, email_data.get("email")) + self.assertIn("subject", email_data) + self.assertIn("message_txt", email_data) return email_data @@ -79,33 +85,32 @@ def _get_email_data(self, include_redirect_url=False): def test_get_verification_email_data_without_verification_url_set(self): email_data = self._get_email_data() self.assertIn( - ('http://testserver/api/v1/profiles/verify_email?' - 'verification_key=%s' % self.verification_key), - email_data.get('message_txt') + ( + "http://testserver/api/v1/profiles/verify_email?" + "verification_key=%s" % self.verification_key + ), + email_data.get("message_txt"), ) @override_settings(VERIFICATION_URL=VERIFICATION_URL) def test_get_verification_email_data_with_verification_url_set(self): email_data = self._get_email_data() self.assertIn( - '{}?verification_key={}'.format( - VERIFICATION_URL, self.verification_key - ), - email_data.get('message_txt') + "{}?verification_key={}".format(VERIFICATION_URL, self.verification_key), + email_data.get("message_txt"), ) @override_settings(VERIFICATION_URL=VERIFICATION_URL) - def test_get_verification_email_data_with_verification_and_redirect_urls( - self): + def test_get_verification_email_data_with_verification_and_redirect_urls(self): email_data = self._get_email_data(include_redirect_url=True) - encoded_url = urlencode({ - 'verification_key': self.verification_key, - 'redirect_url': self.redirect_url - }) - self.assertIn( - encoded_url.replace('&', '&'), email_data.get('message_txt') + encoded_url = urlencode( + { + "verification_key": self.verification_key, + "redirect_url": self.redirect_url, + } ) + self.assertIn(encoded_url.replace("&", "&"), email_data.get("message_txt")) def test_email_data_does_not_contain_newline_chars(self): email_data = self._get_email_data(include_redirect_url=True) - self.assertNotIn('\n', email_data.get('subject')) + self.assertNotIn("\n", email_data.get("subject")) diff --git a/onadata/libs/tests/utils/test_export_builder.py b/onadata/libs/tests/utils/test_export_builder.py index e5c4f8677f..41ea9e8e5b 100644 --- a/onadata/libs/tests/utils/test_export_builder.py +++ b/onadata/libs/tests/utils/test_export_builder.py @@ -10,293 +10,302 @@ import shutil import tempfile import zipfile -from builtins import open from collections import OrderedDict from ctypes import ArgumentError from io import BytesIO +from unittest import skipIf from openpyxl import load_workbook from django.conf import settings from django.core.files.temp import NamedTemporaryFile -from past.builtins import basestring from pyxform.builder import create_survey_from_xls -from savReaderWriter import SavHeaderReader, SavReader + +try: + from savReaderWriter import SavHeaderReader, SavReader +except ImportError: + SavHeaderReader = SavReader = None from onadata.apps.logger.import_tools import django_file from onadata.apps.main.tests.test_base import TestBase from onadata.apps.viewer.models.data_dictionary import DataDictionary -from onadata.apps.viewer.models.parsed_instance import (_encode_for_mongo, - query_data) +from onadata.apps.viewer.models.parsed_instance import _encode_for_mongo, query_data from onadata.apps.viewer.tests.export_helpers import viewer_fixture_path -from onadata.libs.utils.csv_builder import (CSVDataFrameBuilder, - get_labels_from_columns) +from onadata.libs.utils.csv_builder import CSVDataFrameBuilder, get_labels_from_columns from onadata.libs.utils.common_tags import ( - SELECT_BIND_TYPE, MULTIPLE_SELECT_TYPE, REVIEW_COMMENT, REVIEW_DATE, - REVIEW_STATUS) + SELECT_BIND_TYPE, + MULTIPLE_SELECT_TYPE, + REVIEW_COMMENT, + REVIEW_DATE, + REVIEW_STATUS, +) from onadata.libs.utils.export_builder import ( decode_mongo_encoded_section_names, dict_to_joined_export, ExportBuilder, - string_to_date_with_xls_validation) + string_to_date_with_xls_validation, +) from onadata.libs.utils.export_tools import get_columns_with_hxl from onadata.libs.utils.logger_tools import create_instance def _logger_fixture_path(*args): - return os.path.join(settings.PROJECT_ROOT, 'apps', 'logger', - 'tests', 'fixtures', *args) + return os.path.join( + settings.PROJECT_ROOT, "apps", "logger", "tests", "fixtures", *args + ) class TestExportBuilder(TestBase): + """Test onadata.libs.utils.export_builder functions.""" + data = [ { - 'name': 'Abe', - 'age': 35, - 'tel/telLg==office': '020123456', - '_review_status': 'Rejected', - '_review_comment': 'Wrong Location', - REVIEW_DATE: '2021-05-25T02:27:19', - 'children': - [ + "name": "Abe", + "age": 35, + "tel/telLg==office": "020123456", + "_review_status": "Rejected", + "_review_comment": "Wrong Location", + REVIEW_DATE: "2021-05-25T02:27:19", + "children": [ { - 'children/name': 'Mike', - 'children/age': 5, - 'children/fav_colors': 'red blue', - 'children/iceLg==creams': 'vanilla chocolate', - 'children/cartoons': - [ + "children/name": "Mike", + "children/age": 5, + "children/fav_colors": "red blue", + "children/iceLg==creams": "vanilla chocolate", + "children/cartoons": [ { - 'children/cartoons/name': 'Tom & Jerry', - 'children/cartoons/why': 'Tom is silly', + "children/cartoons/name": "Tom & Jerry", + "children/cartoons/why": "Tom is silly", }, { - 'children/cartoons/name': 'Flinstones', - 'children/cartoons/why': u"I like bam bam\u0107" + "children/cartoons/name": "Flinstones", + "children/cartoons/why": "I like bam bam\u0107" # throw in a unicode character - } - ] - }, - { - 'children/name': 'John', - 'children/age': 2, - 'children/cartoons': [] + }, + ], }, + {"children/name": "John", "children/age": 2, "children/cartoons": []}, { - 'children/name': 'Imora', - 'children/age': 3, - 'children/cartoons': - [ + "children/name": "Imora", + "children/age": 3, + "children/cartoons": [ { - 'children/cartoons/name': 'Shrek', - 'children/cartoons/why': 'He\'s so funny' + "children/cartoons/name": "Shrek", + "children/cartoons/why": "He's so funny", }, { - 'children/cartoons/name': 'Dexter\'s Lab', - 'children/cartoons/why': 'He thinks hes smart', - 'children/cartoons/characters': - [ + "children/cartoons/name": "Dexter's Lab", + "children/cartoons/why": "He thinks hes smart", + "children/cartoons/characters": [ { - 'children/cartoons/characters/name': - 'Dee Dee', - 'children/cartoons/characters/good_or_evi' - 'l': 'good' + "children/cartoons/characters/name": "Dee Dee", + "children/cartoons/characters/good_or_evi" + "l": "good", }, { - 'children/cartoons/characters/name': - 'Dexter', - 'children/cartoons/characters/good_or_evi' - 'l': 'evil' + "children/cartoons/characters/name": "Dexter", + "children/cartoons/characters/good_or_evi" + "l": "evil", }, - ] - } - ] - } - ] + ], + }, + ], + }, + ], }, { # blank data just to be sure - 'children': [] - } + "children": [] + }, ] long_survey_data = [ { - 'name': 'Abe', - 'age': 35, - 'childrens_survey_with_a_very_lo': - [ + "name": "Abe", + "age": 35, + "childrens_survey_with_a_very_lo": [ { - 'childrens_survey_with_a_very_lo/name': 'Mike', - 'childrens_survey_with_a_very_lo/age': 5, - 'childrens_survey_with_a_very_lo/fav_colors': 'red blue', - 'childrens_survey_with_a_very_lo/cartoons': - [ + "childrens_survey_with_a_very_lo/name": "Mike", + "childrens_survey_with_a_very_lo/age": 5, + "childrens_survey_with_a_very_lo/fav_colors": "red blue", + "childrens_survey_with_a_very_lo/cartoons": [ { - 'childrens_survey_with_a_very_lo/cartoons/name': - 'Tom & Jerry', - 'childrens_survey_with_a_very_lo/cartoons/why': - 'Tom is silly', + "childrens_survey_with_a_very_lo/cartoons/name": "Tom & Jerry", + "childrens_survey_with_a_very_lo/cartoons/why": "Tom is silly", }, { - 'childrens_survey_with_a_very_lo/cartoons/name': - 'Flinstones', - 'childrens_survey_with_a_very_lo/cartoons/why': - u"I like bam bam\u0107" + "childrens_survey_with_a_very_lo/cartoons/name": "Flinstones", + "childrens_survey_with_a_very_lo/cartoons/why": "I like bam bam\u0107" # throw in a unicode character - } - ] + }, + ], }, { - 'childrens_survey_with_a_very_lo/name': 'John', - 'childrens_survey_with_a_very_lo/age': 2, - 'childrens_survey_with_a_very_lo/cartoons': [] + "childrens_survey_with_a_very_lo/name": "John", + "childrens_survey_with_a_very_lo/age": 2, + "childrens_survey_with_a_very_lo/cartoons": [], }, { - 'childrens_survey_with_a_very_lo/name': 'Imora', - 'childrens_survey_with_a_very_lo/age': 3, - 'childrens_survey_with_a_very_lo/cartoons': - [ + "childrens_survey_with_a_very_lo/name": "Imora", + "childrens_survey_with_a_very_lo/age": 3, + "childrens_survey_with_a_very_lo/cartoons": [ { - 'childrens_survey_with_a_very_lo/cartoons/name': - 'Shrek', - 'childrens_survey_with_a_very_lo/cartoons/why': - 'He\'s so funny' + "childrens_survey_with_a_very_lo/cartoons/name": "Shrek", + "childrens_survey_with_a_very_lo/cartoons/why": "He's so funny", }, { - 'childrens_survey_with_a_very_lo/cartoons/name': - 'Dexter\'s Lab', - 'childrens_survey_with_a_very_lo/cartoons/why': - 'He thinks hes smart', - 'childrens_survey_with_a_very_lo/cartoons/characte' - 'rs': - [ + "childrens_survey_with_a_very_lo/cartoons/name": "Dexter's Lab", + "childrens_survey_with_a_very_lo/cartoons/why": "He thinks hes smart", + "childrens_survey_with_a_very_lo/cartoons/characte" + "rs": [ { - 'childrens_survey_with_a_very_lo/cartoons/' - 'characters/name': 'Dee Dee', - 'children/cartoons/characters/good_or_evi' - 'l': 'good' + "childrens_survey_with_a_very_lo/cartoons/" + "characters/name": "Dee Dee", + "children/cartoons/characters/good_or_evi" + "l": "good", }, { - 'childrens_survey_with_a_very_lo/cartoons/' - 'characters/name': 'Dexter', - 'children/cartoons/characters/good_or_evi' - 'l': 'evil' + "childrens_survey_with_a_very_lo/cartoons/" + "characters/name": "Dexter", + "children/cartoons/characters/good_or_evi" + "l": "evil", }, - ] - } - ] - } - ] + ], + }, + ], + }, + ], } ] data_utf8 = [ { - 'name': 'Abe', - 'age': 35, - 'tel/telLg==office': '020123456', - 'childrenLg==info': - [ + "name": "Abe", + "age": 35, + "tel/telLg==office": "020123456", + "childrenLg==info": [ { - 'childrenLg==info/nameLg==first': 'Mike', - 'childrenLg==info/age': 5, - 'childrenLg==info/fav_colors': "red's blue's", - 'childrenLg==info/ice_creams': 'vanilla chocolate', - 'childrenLg==info/cartoons': - [ + "childrenLg==info/nameLg==first": "Mike", + "childrenLg==info/age": 5, + "childrenLg==info/fav_colors": "red's blue's", + "childrenLg==info/ice_creams": "vanilla chocolate", + "childrenLg==info/cartoons": [ { - 'childrenLg==info/cartoons/name': 'Tom & Jerry', - 'childrenLg==info/cartoons/why': 'Tom is silly', + "childrenLg==info/cartoons/name": "Tom & Jerry", + "childrenLg==info/cartoons/why": "Tom is silly", }, { - 'childrenLg==info/cartoons/name': 'Flinstones', - 'childrenLg==info/cartoons/why': + "childrenLg==info/cartoons/name": "Flinstones", + "childrenLg==info/cartoons/why": # throw in a unicode character - 'I like bam bam\u0107' - } - ] + "I like bam bam\u0107", + }, + ], } - ] + ], } ] osm_data = [ { - 'photo': '1424308569120.jpg', - 'osm_road': 'OSMWay234134797.osm', - 'osm_building': 'OSMWay34298972.osm', - 'fav_color': 'red', - 'osm_road:ctr:lat': '23.708174238006087', - 'osm_road:ctr:lon': '90.40946505581161', - 'osm_road:highway': 'tertiary', - 'osm_road:lanes': 2, - 'osm_road:name': 'Patuatuli Road', - 'osm_building:building': 'yes', - 'osm_building:building:levels': 4, - 'osm_building:ctr:lat': '23.707316084046038', - 'osm_building:ctr:lon': '90.40849938337506', - 'osm_building:name': 'kol', - '_review_status': 'Rejected', - '_review_comment': 'Wrong Location', - REVIEW_DATE: '2021-05-25T02:27:19', + "photo": "1424308569120.jpg", + "osm_road": "OSMWay234134797.osm", + "osm_building": "OSMWay34298972.osm", + "fav_color": "red", + "osm_road:ctr:lat": "23.708174238006087", + "osm_road:ctr:lon": "90.40946505581161", + "osm_road:highway": "tertiary", + "osm_road:lanes": 2, + "osm_road:name": "Patuatuli Road", + "osm_building:building": "yes", + "osm_building:building:levels": 4, + "osm_building:ctr:lat": "23.707316084046038", + "osm_building:ctr:lon": "90.40849938337506", + "osm_building:name": "kol", + "_review_status": "Rejected", + "_review_comment": "Wrong Location", + REVIEW_DATE: "2021-05-25T02:27:19", } ] def _create_childrens_survey(self, filename="childrens_survey.xlsx"): - survey = create_survey_from_xls(_logger_fixture_path( - filename - ), default_name=filename.split('.')[0]) - self.dd = DataDictionary() - self.dd._survey = survey + survey = create_survey_from_xls( + _logger_fixture_path(filename), default_name=filename.split(".")[0] + ) + self.data_dictionary = DataDictionary() + setattr(self.data_dictionary, "_survey", survey) return survey + # pylint: disable=invalid-name def test_build_sections_for_multilanguage_form(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'multi_lingual_form.xlsx'), - default_name='multi_lingual_form') + survey = create_survey_from_xls( + _logger_fixture_path("multi_lingual_form.xlsx"), + default_name="multi_lingual_form", + ) # check the default langauge - self.assertEqual( - survey.to_json_dict().get('default_language'), 'English' - ) + self.assertEqual(survey.to_json_dict().get("default_language"), "English") export_builder = ExportBuilder() export_builder.INCLUDE_LABELS_ONLY = True export_builder.set_survey(survey) - expected_sections = [ - survey.name] + expected_sections = [survey.name] self.assertEqual( - expected_sections, [s['name'] for s in export_builder.sections]) - expected_element_names = \ - ['Name of respondent', 'Age', 'Sex of respondent', 'Fruits', - 'Fruits/Apple', 'Fruits/Banana', 'Fruits/Pear', 'Fruits/Mango', - 'Fruits/Other', 'Fruits/None of the above', 'Cities', - 'meta/instanceID'] + expected_sections, [s["name"] for s in export_builder.sections] + ) + expected_element_names = [ + "Name of respondent", + "Age", + "Sex of respondent", + "Fruits", + "Fruits/Apple", + "Fruits/Banana", + "Fruits/Pear", + "Fruits/Mango", + "Fruits/Other", + "Fruits/None of the above", + "Cities", + "meta/instanceID", + ] section = export_builder.section_by_name(survey.name) - element_names = [element['label'] for element in section['elements']] - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + element_names = [element["label"] for element in section["elements"]] + self.assertEqual(sorted(expected_element_names), sorted(element_names)) - export_builder.language = 'French' + export_builder.language = "French" export_builder.set_survey(survey) section = export_builder.section_by_name(survey.name) - element_names = [element['label'] for element in section['elements']] - expected_element_names = \ - ['Des fruits', 'Fruits/Aucune de ces réponses', 'Fruits/Autre', - 'Fruits/Banane', 'Fruits/Mangue', 'Fruits/Poire', 'Fruits/Pomme', - "L'age", 'Le genre', 'Nom de personne interrogée', 'Villes', - 'meta/instanceID'] - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + element_names = [element["label"] for element in section["elements"]] + expected_element_names = [ + "Des fruits", + "Fruits/Aucune de ces réponses", + "Fruits/Autre", + "Fruits/Banane", + "Fruits/Mangue", + "Fruits/Poire", + "Fruits/Pomme", + "L'age", + "Le genre", + "Nom de personne interrogée", + "Villes", + "meta/instanceID", + ] + self.assertEqual(sorted(expected_element_names), sorted(element_names)) # use default language when the language passed does not exist - export_builder.language = 'Kiswahili' + export_builder.language = "Kiswahili" export_builder.set_survey(survey) - expected_element_names = \ - ['Name of respondent', 'Age', 'Sex of respondent', 'Fruits', - 'Fruits/Apple', 'Fruits/Banana', 'Fruits/Pear', 'Fruits/Mango', - 'Fruits/Other', 'Fruits/None of the above', 'Cities', - 'meta/instanceID'] + expected_element_names = [ + "Name of respondent", + "Age", + "Sex of respondent", + "Fruits", + "Fruits/Apple", + "Fruits/Banana", + "Fruits/Pear", + "Fruits/Mango", + "Fruits/Other", + "Fruits/None of the above", + "Cities", + "meta/instanceID", + ] section = export_builder.section_by_name(survey.name) - element_names = [element['label'] for element in section['elements']] - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + element_names = [element["label"] for element in section["elements"]] + self.assertEqual(sorted(expected_element_names), sorted(element_names)) def test_build_sections_from_survey(self): survey = self._create_childrens_survey() @@ -304,62 +313,73 @@ def test_build_sections_from_survey(self): export_builder.set_survey(survey) # test that we generate the proper sections expected_sections = [ - survey.name, 'children', 'children/cartoons', - 'children/cartoons/characters'] + survey.name, + "children", + "children/cartoons", + "children/cartoons/characters", + ] self.assertEqual( - expected_sections, [s['name'] for s in export_builder.sections]) + expected_sections, [s["name"] for s in export_builder.sections] + ) # main section should have split geolocations expected_element_names = [ - 'name', 'age', 'geo/geolocation', 'geo/_geolocation_longitude', - 'geo/_geolocation_latitude', 'geo/_geolocation_altitude', - 'geo/_geolocation_precision', 'tel/tel.office', 'tel/tel.mobile', - 'meta/instanceID'] + "name", + "age", + "geo/geolocation", + "geo/_geolocation_longitude", + "geo/_geolocation_latitude", + "geo/_geolocation_altitude", + "geo/_geolocation_precision", + "tel/tel.office", + "tel/tel.mobile", + "meta/instanceID", + ] section = export_builder.section_by_name(survey.name) - element_names = [element['xpath'] for element in section['elements']] + element_names = [element["xpath"] for element in section["elements"]] # fav_colors should have its choices split - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + self.assertEqual(sorted(expected_element_names), sorted(element_names)) expected_element_names = [ - 'children/name', 'children/age', 'children/fav_colors', - 'children/fav_colors/red', 'children/fav_colors/blue', - 'children/fav_colors/pink', 'children/ice.creams', - 'children/ice.creams/vanilla', 'children/ice.creams/strawberry', - 'children/ice.creams/chocolate'] - section = export_builder.section_by_name('children') - element_names = [element['xpath'] for element in section['elements']] - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + "children/name", + "children/age", + "children/fav_colors", + "children/fav_colors/red", + "children/fav_colors/blue", + "children/fav_colors/pink", + "children/ice.creams", + "children/ice.creams/vanilla", + "children/ice.creams/strawberry", + "children/ice.creams/chocolate", + ] + section = export_builder.section_by_name("children") + element_names = [element["xpath"] for element in section["elements"]] + self.assertEqual(sorted(expected_element_names), sorted(element_names)) - expected_element_names = [ - 'children/cartoons/name', 'children/cartoons/why'] - section = export_builder.section_by_name('children/cartoons') - element_names = [element['xpath'] for element in section['elements']] + expected_element_names = ["children/cartoons/name", "children/cartoons/why"] + section = export_builder.section_by_name("children/cartoons") + element_names = [element["xpath"] for element in section["elements"]] - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + self.assertEqual(sorted(expected_element_names), sorted(element_names)) expected_element_names = [ - 'children/cartoons/characters/name', - 'children/cartoons/characters/good_or_evil'] - section = \ - export_builder.section_by_name('children/cartoons/characters') - element_names = [element['xpath'] for element in section['elements']] - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + "children/cartoons/characters/name", + "children/cartoons/characters/good_or_evil", + ] + section = export_builder.section_by_name("children/cartoons/characters") + element_names = [element["xpath"] for element in section["elements"]] + self.assertEqual(sorted(expected_element_names), sorted(element_names)) + # pylint: disable=too-many-locals def test_zipped_csv_export_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - export_builder.to_zipped_csv(temp_zip_file.name, self.data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, 'r') - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_csv(temp_zip_file.name, self.data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # generate data to compare with index = 1 @@ -368,70 +388,66 @@ def test_zipped_csv_export_works(self): outputs = [] for d in self.data: outputs.append( - dict_to_joined_export( - d, index, indices, survey_name, survey, d)) + dict_to_joined_export(d, index, indices, survey_name, survey, d) + ) index += 1 # check that each file exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "{0}.csv".format(survey.name)))) - with open(os.path.join(temp_dir, "{0}.csv".format(survey.name)), - encoding='utf-8') as csv_file: + self.assertTrue(os.path.exists(os.path.join(temp_dir, f"{survey.name}.csv"))) + with open( + os.path.join(temp_dir, f"{survey.name}.csv"), encoding="utf-8" + ) as csv_file: reader = csv.reader(csv_file) - rows = [r for r in reader] + rows = list(reader) # open comparison file - with open(_logger_fixture_path('csvs', 'childrens_survey.csv'), - encoding='utf-8') as fixture_csv: - fixture_reader = csv.reader(fixture_csv) - expected_rows = [r for r in fixture_reader] + with open( + _logger_fixture_path("csvs", "childrens_survey.csv"), encoding="utf-8" + ) as fixture_csv: + expected_rows = list(csv.reader(fixture_csv)) self.assertEqual(rows, expected_rows) - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children.csv"))) - with open(os.path.join(temp_dir, "children.csv"), - encoding='utf-8') as csv_file: + self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.csv"))) + with open(os.path.join(temp_dir, "children.csv"), encoding="utf-8") as csv_file: reader = csv.reader(csv_file) - rows = [r for r in reader] + rows = list(reader) # open comparison file - with open(_logger_fixture_path('csvs', 'children.csv'), - encoding='utf-8') as fixture_csv: - fixture_reader = csv.reader(fixture_csv) - expected_rows = [r for r in fixture_reader] + with open( + _logger_fixture_path("csvs", "children.csv"), encoding="utf-8" + ) as fixture_csv: + expected_rows = list(csv.reader(fixture_csv)) self.assertEqual(rows, expected_rows) - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children_cartoons.csv"))) - with open(os.path.join(temp_dir, "children_cartoons.csv"), - encoding='utf-8') as csv_file: + self.assertTrue(os.path.exists(os.path.join(temp_dir, "children_cartoons.csv"))) + with open( + os.path.join(temp_dir, "children_cartoons.csv"), encoding="utf-8" + ) as csv_file: reader = csv.reader(csv_file) - rows = [r for r in reader] + rows = list(reader) # open comparison file - with open(_logger_fixture_path('csvs', 'children_cartoons.csv'), - encoding='utf-8') as fixture_csv: - fixture_reader = csv.reader(fixture_csv) - expected_rows = [r for r in fixture_reader] + with open( + _logger_fixture_path("csvs", "children_cartoons.csv"), encoding="utf-8" + ) as fixture_csv: + expected_rows = list(csv.reader(fixture_csv)) self.assertEqual(rows, expected_rows) self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children_cartoons_characters.csv"))) - with open(os.path.join(temp_dir, "children_cartoons_characters.csv"), - encoding='utf-8') as csv_file: + os.path.exists(os.path.join(temp_dir, "children_cartoons_characters.csv")) + ) + with open( + os.path.join(temp_dir, "children_cartoons_characters.csv"), encoding="utf-8" + ) as csv_file: reader = csv.reader(csv_file) - rows = [r for r in reader] + rows = list(reader) # open comparison file - with open(_logger_fixture_path( - 'csvs', 'children_cartoons_characters.csv'), - encoding='utf-8') as fixture_csv: - fixture_reader = csv.reader(fixture_csv) - expected_rows = [r for r in fixture_reader] + with open( + _logger_fixture_path("csvs", "children_cartoons_characters.csv"), + encoding="utf-8", + ) as fixture_csv: + expected_rows = list(csv.reader(fixture_csv)) self.assertEqual(rows, expected_rows) shutil.rmtree(temp_dir) @@ -444,98 +460,128 @@ def test_xls_export_with_osm_data(self): xform = self.xform export_builder = ExportBuilder() export_builder.set_survey(survey, xform) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - export_builder.to_xls_export(temp_xls_file.name, self.osm_data) - temp_xls_file.seek(0) - wb = load_workbook(temp_xls_file.name) - osm_data_sheet = wb["osm"] - rows = [row for row in osm_data_sheet.rows] - xls_headers = [a.value for a in rows[0]] - temp_xls_file.close() + with NamedTemporaryFile(suffix=".xlsx") as temp_xls_file: + export_builder.to_xls_export(temp_xls_file.name, self.osm_data) + temp_xls_file.seek(0) + workbook = load_workbook(temp_xls_file.name) + osm_data_sheet = workbook["osm"] + rows = list(osm_data_sheet.rows) + xls_headers = [a.value for a in rows[0]] expected_column_headers = [ - 'photo', 'osm_road', 'osm_building', 'fav_color', - 'form_completed', 'meta/instanceID', '_id', '_uuid', - '_submission_time', '_index', '_parent_table_name', - '_parent_index', '_tags', '_notes', '_version', '_duration', - '_submitted_by', 'osm_road:ctr:lat', 'osm_road:ctr:lon', - 'osm_road:highway', 'osm_road:lanes', 'osm_road:name', - 'osm_road:way:id', 'osm_building:building', - 'osm_building:building:levels', 'osm_building:ctr:lat', - 'osm_building:ctr:lon', 'osm_building:name', - 'osm_building:way:id'] + "photo", + "osm_road", + "osm_building", + "fav_color", + "form_completed", + "meta/instanceID", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + "osm_road:ctr:lat", + "osm_road:ctr:lon", + "osm_road:highway", + "osm_road:lanes", + "osm_road:name", + "osm_road:way:id", + "osm_building:building", + "osm_building:building:levels", + "osm_building:ctr:lat", + "osm_building:ctr:lon", + "osm_building:name", + "osm_building:way:id", + ] self.assertEqual(sorted(expected_column_headers), sorted(xls_headers)) submission = [a.value for a in rows[1]] - self.assertEqual(submission[0], '1424308569120.jpg') - self.assertEqual(submission[2], '23.708174238006087') - self.assertEqual(submission[4], 'tertiary') - self.assertEqual(submission[6], 'Patuatuli Road') - self.assertEqual(submission[11], '23.707316084046038') - self.assertEqual(submission[13], 'kol') - + self.assertEqual(submission[0], "1424308569120.jpg") + self.assertEqual(submission[2], "23.708174238006087") + self.assertEqual(submission[4], "tertiary") + self.assertEqual(submission[6], "Patuatuli Road") + self.assertEqual(submission[11], "23.707316084046038") + self.assertEqual(submission[13], "kol") + + # pylint: disable=invalid-name def test_decode_mongo_encoded_section_names(self): data = { - 'main_section': [1, 2, 3, 4], - 'sectionLg==1/info': [1, 2, 3, 4], - 'sectionLg==2/info': [1, 2, 3, 4], + "main_section": [1, 2, 3, 4], + "sectionLg==1/info": [1, 2, 3, 4], + "sectionLg==2/info": [1, 2, 3, 4], } result = decode_mongo_encoded_section_names(data) expected_result = { - 'main_section': [1, 2, 3, 4], - 'section.1/info': [1, 2, 3, 4], - 'section.2/info': [1, 2, 3, 4], + "main_section": [1, 2, 3, 4], + "section.1/info": [1, 2, 3, 4], + "section.2/info": [1, 2, 3, 4], } self.assertEqual(result, expected_result) + # pylint: disable=invalid-name def test_zipped_csv_export_works_with_unicode(self): """ cvs writer doesnt handle unicode we we have to encode to ascii """ - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xlsx'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xlsx"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children.info.csv"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.info.csv"))) # check file's contents - with open(os.path.join(temp_dir, "children.info.csv"), - encoding='utf-8') as csv_file: + with open( + os.path.join(temp_dir, "children.info.csv"), encoding="utf-8" + ) as csv_file: reader = csv.reader(csv_file) - expected_headers = ['children.info/name.first', - 'children.info/age', - 'children.info/fav_colors', - 'children.info/fav_colors/red\'s', - 'children.info/fav_colors/blue\'s', - 'children.info/fav_colors/pink\'s', - 'children.info/ice_creams', - 'children.info/ice_creams/vanilla', - 'children.info/ice_creams/strawberry', - 'children.info/ice_creams/chocolate', '_id', - '_uuid', '_submission_time', '_index', - '_parent_table_name', '_parent_index', - '_tags', '_notes', '_version', - '_duration', '_submitted_by'] - rows = [row for row in reader] - actual_headers = [h for h in rows[0]] + expected_headers = [ + "children.info/name.first", + "children.info/age", + "children.info/fav_colors", + "children.info/fav_colors/red's", + "children.info/fav_colors/blue's", + "children.info/fav_colors/pink's", + "children.info/ice_creams", + "children.info/ice_creams/vanilla", + "children.info/ice_creams/strawberry", + "children.info/ice_creams/chocolate", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + rows = list(reader) + actual_headers = list(rows[0]) self.assertEqual(sorted(actual_headers), sorted(expected_headers)) data = dict(zip(rows[0], rows[1])) - self.assertEqual(data['children.info/fav_colors/red\'s'], 'True') - self.assertEqual(data['children.info/fav_colors/blue\'s'], 'True') - self.assertEqual(data['children.info/fav_colors/pink\'s'], 'False') + self.assertEqual(data["children.info/fav_colors/red's"], "True") + self.assertEqual(data["children.info/fav_colors/blue's"], "True") + self.assertEqual(data["children.info/fav_colors/pink's"], "False") # check that red and blue are set to true + # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_date_field(self): md = """ | survey | @@ -548,38 +594,40 @@ def test_zipped_sav_export_with_date_field(self): | choices | | | list name | name | label | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{"expense_date": "2013-01-03", "A/gdate": "2017-06-13", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [ + { + "expense_date": "2013-01-03", + "A/gdate": "2017-06-13", + "_submission_time": "2016-11-21T03:43:43.000-08:00", + } + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, "exp.sav"), - returnHeader=True) as reader: - rows = [r for r in reader] + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: + rows = list(reader) self.assertTrue(len(rows) > 1) - self.assertEqual(rows[0][0], b'expense_date') - self.assertEqual(rows[1][0], b'2013-01-03') - self.assertEqual(rows[0][1], b'A.gdate') - self.assertEqual(rows[1][1], b'2017-06-13') - self.assertEqual(rows[0][5], b'@_submission_time') - self.assertEqual(rows[1][5], b'2016-11-21 03:43:43') + self.assertEqual(rows[0][0], b"expense_date") + self.assertEqual(rows[1][0], b"2013-01-03") + self.assertEqual(rows[0][1], b"A.gdate") + self.assertEqual(rows[1][1], b"2017-06-13") + self.assertEqual(rows[0][5], b"@_submission_time") + self.assertEqual(rows[1][5], b"2016-11-21 03:43:43") shutil.rmtree(temp_dir) + # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_dynamic_select_multiple(self): md = """ | survey | @@ -599,50 +647,53 @@ def test_zipped_sav_export_dynamic_select_multiple(self): | | brand | a | a | | | brand | b | b | """ # noqa - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) data = [ - {"sex": "male", "text": "his", "favorite_brand": "Generic", - "name": "Davis", "brand_known": "${text} ${favorite_brand} a"}] + { + "sex": "male", + "text": "his", + "favorite_brand": "Generic", + "name": "Davis", + "brand_known": "${text} ${favorite_brand} a", + } + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) - with SavReader(os.path.join(temp_dir, "exp.sav"), - returnHeader=True) as reader: - rows = [r for r in reader] + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: + rows = list(reader) self.assertTrue(len(rows) > 1) - self.assertEqual(rows[0][0], b'sex') - self.assertEqual(rows[1][0], b'male') - self.assertEqual(rows[0][1], b'text') - self.assertEqual(rows[1][1], b'his') - self.assertEqual(rows[0][2], b'favorite_brand') - self.assertEqual(rows[1][2], b'Generic') - self.assertEqual(rows[0][3], b'name') - self.assertEqual(rows[1][3], b'Davis') - self.assertEqual(rows[0][4], b'brand_known') - self.assertEqual(rows[1][4], b'his Generic a') - self.assertEqual(rows[0][5], b'brand_known.$text') + self.assertEqual(rows[0][0], b"sex") + self.assertEqual(rows[1][0], b"male") + self.assertEqual(rows[0][1], b"text") + self.assertEqual(rows[1][1], b"his") + self.assertEqual(rows[0][2], b"favorite_brand") + self.assertEqual(rows[1][2], b"Generic") + self.assertEqual(rows[0][3], b"name") + self.assertEqual(rows[1][3], b"Davis") + self.assertEqual(rows[0][4], b"brand_known") + self.assertEqual(rows[1][4], b"his Generic a") + self.assertEqual(rows[0][5], b"brand_known.$text") self.assertEqual(rows[1][5], 1.0) - self.assertEqual(rows[0][6], b'brand_known.$favorite_brand') + self.assertEqual(rows[0][6], b"brand_known.$favorite_brand") self.assertEqual(rows[1][6], 1.0) - self.assertEqual(rows[0][7], b'brand_known.a') + self.assertEqual(rows[0][7], b"brand_known.a") self.assertEqual(rows[1][7], 1.0) - self.assertEqual(rows[0][8], b'brand_known.b') + self.assertEqual(rows[0][8], b"brand_known.b") self.assertEqual(rows[1][8], 0.0) shutil.rmtree(temp_dir) + # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_zero_padded_select_one_field(self): md = """ | survey | @@ -654,32 +705,28 @@ def test_zipped_sav_export_with_zero_padded_select_one_field(self): | | yes_no | 1 | Yes | | | yes_no | 09 | No | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{'expensed': '09', - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [{"expensed": "09", "_submission_time": "2016-11-21T03:43:43.000-08:00"}] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, 'r') - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, 'exp.sav'))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, 'exp.sav'), - returnHeader=True) as reader: - rows = [r for r in reader] + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: + rows = list(reader) self.assertTrue(len(rows) > 1) - self.assertEqual(rows[1][0].decode('utf-8'), '09') - self.assertEqual(rows[1][4].decode('utf-8'), '2016-11-21 03:43:43') + self.assertEqual(rows[1][0].decode("utf-8"), "09") + self.assertEqual(rows[1][4].decode("utf-8"), "2016-11-21 03:43:43") + # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_numeric_select_one_field(self): md = """ | survey | @@ -694,42 +741,44 @@ def test_zipped_sav_export_with_numeric_select_one_field(self): | | yes_no | 1 | Yes | | | yes_no | 0 | No | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{"expensed": "1", "A/q1": "1", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [ + { + "expensed": "1", + "A/q1": "1", + "_submission_time": "2016-11-21T03:43:43.000-08:00", + } + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, "exp.sav"), - returnHeader=True) as reader: - rows = [r for r in reader] + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: + rows = list(reader) self.assertTrue(len(rows) > 1) # expensed 1 - self.assertEqual(rows[0][0], b'expensed') + self.assertEqual(rows[0][0], b"expensed") self.assertEqual(rows[1][0], 1) # A/q1 1 - self.assertEqual(rows[0][1], b'A.q1') + self.assertEqual(rows[0][1], b"A.q1") self.assertEqual(rows[1][1], 1) # _submission_time is a date string - self.assertEqual(rows[0][5], b'@_submission_time') - self.assertEqual(rows[1][5], b'2016-11-21 03:43:43') + self.assertEqual(rows[0][5], b"@_submission_time") + self.assertEqual(rows[1][5], b"2016-11-21 03:43:43") + # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_duplicate_field_different_groups(self): """ Test SAV exports duplicate fields, same group - one field in repeat @@ -759,26 +808,28 @@ def test_zipped_sav_export_with_duplicate_field_different_groups(self): | | x_y | 2 | Non | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) - labels = export_builder._get_sav_value_labels({'A/allaite': 'allaite'}) - self.assertEqual(labels, {'allaite': {'1': 'Oui', '2': 'Non'}}) + # pylint: disable=protected-access + labels = export_builder._get_sav_value_labels({"A/allaite": "allaite"}) + self.assertEqual(labels, {"allaite": {"1": "Oui", "2": "Non"}}) + # pylint: disable=protected-access repeat_group_labels = export_builder._get_sav_value_labels( - {'A/rep/allaite': 'allaite'}) - self.assertEqual(repeat_group_labels, - {'allaite': {1: 'Yes', 2: 'No'}}) - - temp_zip_file = NamedTemporaryFile(suffix='.zip') + {"A/rep/allaite": "allaite"} + ) + self.assertEqual(repeat_group_labels, {"allaite": {1: "Yes", 2: "No"}}) - try: - export_builder.to_zipped_sav(temp_zip_file.name, []) - except ArgumentError as e: - self.fail(e) + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + try: + export_builder.to_zipped_sav(temp_zip_file.name, []) + except ArgumentError as e: + self.fail(e) + # pylint: disable=invalid-name def test_split_select_multiples_choices_with_randomize_param(self): """ Test that `_get_select_mulitples_choices` function generates @@ -812,57 +863,62 @@ def test_split_select_multiples_choices_with_randomize_param(self): | | allow_choice_duplicates | | | Yes | """ # noqa: E501 - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) dd = DataDictionary() + # pylint: disable=protected-access dd._survey = survey export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - child = [e for e in dd.get_survey_elements_with_choices() - if e.bind.get('type') == SELECT_BIND_TYPE - and e.type == MULTIPLE_SELECT_TYPE][0] + child = [ + e + for e in dd.get_survey_elements_with_choices() + if e.bind.get("type") == SELECT_BIND_TYPE and e.type == MULTIPLE_SELECT_TYPE + ][0] + # pylint: disable=protected-access choices = export_builder._get_select_mulitples_choices( - child, dd, ExportBuilder.GROUP_DELIMITER, - ExportBuilder.TRUNCATE_GROUP_TITLE + child, dd, ExportBuilder.GROUP_DELIMITER, ExportBuilder.TRUNCATE_GROUP_TITLE ) expected_choices = [ { - '_label': 'King', - '_label_xpath': 'county/King', - 'label': 'county/King', - 'title': 'county/king', - 'type': 'string', - 'xpath': 'county/king' + "_label": "King", + "_label_xpath": "county/King", + "label": "county/King", + "title": "county/king", + "type": "string", + "xpath": "county/king", }, { - '_label': 'Pierce', - '_label_xpath': 'county/Pierce', - 'label': 'county/Pierce', - 'title': 'county/pierce', - 'type': 'string', - 'xpath': 'county/pierce' + "_label": "Pierce", + "_label_xpath": "county/Pierce", + "label": "county/Pierce", + "title": "county/pierce", + "type": "string", + "xpath": "county/pierce", }, { - '_label': 'King', - '_label_xpath': 'county/King', - 'label': 'county/King', - 'title': 'county/king', - 'type': 'string', - 'xpath': 'county/king' + "_label": "King", + "_label_xpath": "county/King", + "label": "county/King", + "title": "county/king", + "type": "string", + "xpath": "county/king", }, { - '_label': 'Cameron', - '_label_xpath': 'county/Cameron', - 'label': 'county/Cameron', - 'title': 'county/cameron', - 'type': 'string', - 'xpath': 'county/cameron' - } + "_label": "Cameron", + "_label_xpath": "county/Cameron", + "label": "county/Cameron", + "title": "county/cameron", + "type": "string", + "xpath": "county/cameron", + }, ] self.assertEqual(choices, expected_choices) + # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_numeric_select_multiple_field(self): md = """ | survey | | | | | @@ -879,28 +935,29 @@ def test_zipped_sav_export_with_numeric_select_multiple_field(self): | | y_n | 0 | No | | | | | | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{"expensed": "1", "A/q1": "1", "A/q2": "1", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [ + { + "expensed": "1", + "A/q1": "1", + "A/q2": "1", + "_submission_time": "2016-11-21T03:43:43.000-08:00", + } + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, "exp.sav"), - returnHeader=True) as reader: - rows = [r for r in reader] + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: + rows = list(reader) self.assertTrue(len(rows) > 1) self.assertEqual(rows[0][0], b"expensed") @@ -931,10 +988,12 @@ def test_zipped_sav_export_with_numeric_select_multiple_field(self): self.assertEqual(rows[1][5], 0) self.assertEqual(rows[0][12], b"@_submission_time") - self.assertEqual(rows[1][12], b'2016-11-21 03:43:43') + self.assertEqual(rows[1][12], b"2016-11-21 03:43:43") shutil.rmtree(temp_dir) + # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_zero_padded_select_multiple_field(self): md = """ | survey | | | | @@ -946,38 +1005,34 @@ def test_zipped_sav_export_with_zero_padded_select_multiple_field(self): | | yes_no | 1 | Yes | | | yes_no | 09 | No | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{"expensed": "1", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [{"expensed": "1", "_submission_time": "2016-11-21T03:43:43.000-08:00"}] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, "exp.sav"), - returnHeader=True) as reader: - rows = [r for r in reader] + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: + rows = list(reader) self.assertTrue(len(rows) > 1) self.assertEqual(rows[1][0], b"1") # expensed.1 is selected hence True, 1.00 or 1 in SPSS self.assertEqual(rows[1][1], 1) # expensed.0 is not selected hence False, .00 or 0 in SPSS self.assertEqual(rows[1][2], 0) - self.assertEqual(rows[1][6], b'2016-11-21 03:43:43') + self.assertEqual(rows[1][6], b"2016-11-21 03:43:43") shutil.rmtree(temp_dir) + # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_values_split_select_multiple(self): md = """ | survey | | | | @@ -989,39 +1044,37 @@ def test_zipped_sav_export_with_values_split_select_multiple(self): | | yes_no | 2 | Yes | | | yes_no | 09 | No | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{"expensed": "2 09", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [ + {"expensed": "2 09", "_submission_time": "2016-11-21T03:43:43.000-08:00"} + ] export_builder = ExportBuilder() export_builder.VALUE_SELECT_MULTIPLES = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, "exp.sav"), - returnHeader=True) as reader: - rows = [r for r in reader] + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: + rows = list(reader) self.assertTrue(len(rows) > 1) self.assertEqual(rows[1][0], b"2 09") # expensed.1 is selected hence True, 1.00 or 1 in SPSS self.assertEqual(rows[1][1], 2) # expensed.0 is not selected hence False, .00 or 0 in SPSS - self.assertEqual(rows[1][2], b'09') - self.assertEqual(rows[1][6], b'2016-11-21 03:43:43') + self.assertEqual(rows[1][2], b"09") + self.assertEqual(rows[1][6], b"2016-11-21 03:43:43") shutil.rmtree(temp_dir) + # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_duplicate_name_in_choice_list(self): md = """ | survey | | | | @@ -1044,28 +1097,25 @@ def test_zipped_sav_export_with_duplicate_name_in_choice_list(self): | | allow_choice_duplicates | | | Yes | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{"q1": "1", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}, - {"q1": "6", - '_submission_time': '2016-11-21T03:43:43.000-08:00'} - ] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [ + {"q1": "1", "_submission_time": "2016-11-21T03:43:43.000-08:00"}, + {"q1": "6", "_submission_time": "2016-11-21T03:43:43.000-08:00"}, + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) - def test_zipped_sav_export_external_choices(self): # pylint: disable=C0103 + # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") + def test_zipped_sav_export_external_choices(self): # pylint: disable=invalid-name """ Test that an SPSS export does not fail when it has choices from a file. """ @@ -1074,26 +1124,24 @@ def test_zipped_sav_export_external_choices(self): # pylint: disable=C0103 | | type | name | label | | | select_one_from_file animals.csv | q1 | Favorite animal? | """ - survey = self.md_to_pyxform_survey(xform_markdown, {'name': 'exp'}) - data = [{"q1": "1", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}, - {"q1": "6", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(xform_markdown, {"name": "exp"}) + data = [ + {"q1": "1", "_submission_time": "2016-11-21T03:43:43.000-08:00"}, + {"q1": "6", "_submission_time": "2016-11-21T03:43:43.000-08:00"}, + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) + # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_duplicate_column_name(self): """ Test that SAV exports with duplicate column names @@ -1104,29 +1152,31 @@ def test_zipped_sav_export_with_duplicate_column_name(self): | | text | Sport | Which sport | | | text | sport | Which fun sports| """ - survey = self.md_to_pyxform_survey(md, {'name': 'sports'}) - data = [{"Sport": "Basketball", "sport": "Soccer", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "sports"}) + data = [ + { + "Sport": "Basketball", + "sport": "Soccer", + "_submission_time": "2016-11-21T03:43:43.000-08:00", + } + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "sports.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "sports.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, "sports.sav"), - returnHeader=True) as reader: - rows = [r for r in reader] + with SavReader( + os.path.join(temp_dir, "sports.sav"), returnHeader=True + ) as reader: + rows = list(reader) # Check that columns are present self.assertIn(b"Sport", rows[0]) @@ -1134,41 +1184,49 @@ def test_zipped_sav_export_with_duplicate_column_name(self): # because rows contains 'sport@d4b6' self.assertIn(b"sport", [x[0:5] for x in rows[0]]) + # pylint: disable=invalid-name def test_xls_export_works_with_unicode(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xlsx'), - default_name='childrenss_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xlsx"), + default_name="childrenss_survey_unicode", + ) export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) - temp_xls_file.seek(0) - # check that values for red\'s and blue\'s are set to true - wb = load_workbook(temp_xls_file.name) - children_sheet = wb["children.info"] - data = dict([(x.value, y.value) for x, y in children_sheet.columns]) - self.assertTrue(data["children.info/fav_colors/red's"]) - self.assertTrue(data["children.info/fav_colors/blue's"]) - self.assertFalse(data["children.info/fav_colors/pink's"]) - temp_xls_file.close() - + with NamedTemporaryFile(suffix=".xlsx") as temp_xls_file: + export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) + temp_xls_file.seek(0) + # check that values for red\'s and blue\'s are set to true + workbook = load_workbook(temp_xls_file.name) + children_sheet = workbook["children.info"] + data = {x.value: y.value for x, y in children_sheet.columns} + self.assertTrue(data["children.info/fav_colors/red's"]) + self.assertTrue(data["children.info/fav_colors/blue's"]) + self.assertFalse(data["children.info/fav_colors/pink's"]) + + # pylint: disable=invalid-name def test_xls_export_with_hxl_adds_extra_row(self): # hxl_example.xlsx contains `instance::hxl` column whose value is #age xlsform_path = os.path.join( - settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", - "hxl_test", "hxl_example.xlsx") + settings.PROJECT_ROOT, + "apps", + "main", + "tests", + "fixtures", + "hxl_test", + "hxl_example.xlsx", + ) survey = create_survey_from_xls( - xlsform_path, - default_name=xlsform_path.split('/')[-1].split('.')[0]) + xlsform_path, default_name=xlsform_path.split("/")[-1].split(".")[0] + ) export_builder = ExportBuilder() export_builder.INCLUDE_HXL = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") survey_elements = [ survey_item[1] for survey_item in survey.items() - if survey_item[0] == 'children' + if survey_item[0] == "children" ][0] columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl( @@ -1176,18 +1234,19 @@ def test_xls_export_with_hxl_adds_extra_row(self): ) export_builder.to_xls_export( - temp_xls_file.name, self.data_utf8, - columns_with_hxl=columns_with_hxl) + temp_xls_file.name, self.data_utf8, columns_with_hxl=columns_with_hxl + ) temp_xls_file.seek(0) - wb = load_workbook(temp_xls_file.name) - children_sheet = wb["hxl_example"] + workbook = load_workbook(temp_xls_file.name) + children_sheet = workbook["hxl_example"] self.assertTrue(children_sheet) # we pick the second row because the first row has xform fieldnames - rows = [row for row in children_sheet.rows] + rows = list(children_sheet.rows) hxl_row = [a.value for a in rows[1]] - self.assertIn('#age', hxl_row) + self.assertIn("#age", hxl_row) + # pylint: disable=invalid-name,too-many-locals def test_export_with_image_attachments(self): """ Test that the url for images is displayed correctly in exports @@ -1200,539 +1259,600 @@ def test_export_with_image_attachments(self): self._create_user_and_login() self.xform = self._publish_markdown(md, self.user) - xml_string = """ - + xml_string = f""" + uuid:UJ6jSMAJ1Jz4EszdgHy8n852AsKaqBPO5 1300221157303.jpg - """.format(self.xform.id_string) - - file_path = "{}/apps/logger/tests/Health_2011_03_13."\ - "xml_2011-03-15_20-30-28/1300221157303"\ - ".jpg".format(settings.PROJECT_ROOT) - media_file = django_file(path=file_path, - field_name="image1", - content_type="image/jpeg") - create_instance(self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media_file]) + """ + + file_path = ( + f"{settings.PROJECT_ROOT}/apps/logger/tests/Health_2011_03_13." + "xml_2011-03-15_20-30-28/1300221157303.jpg" + ) + media_file = django_file( + path=file_path, field_name="image1", content_type="image/jpeg" + ) + create_instance( + self.user.username, + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media_file], + ) xdata = query_data(self.xform) - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - export_builder.to_xls_export(temp_xls_file, xdata) - temp_xls_file.seek(0) - wb = load_workbook(temp_xls_file) - children_sheet = wb["exp"] - self.assertTrue(children_sheet) - rows = [row for row in children_sheet.rows] - row = [a.value for a in rows[1]] - attachment_id = xdata[0]['_attachments'][0]['id'] - attachment_filename = xdata[0]['_attachments'][0]['filename'] - attachment_url = 'http://example.com/api/v1/files/{}?filename={}'.format(attachment_id, attachment_filename) # noqa - self.assertIn(attachment_url, row) - temp_xls_file.close() - + with NamedTemporaryFile(suffix=".xlsx") as temp_xls_file: + export_builder.to_xls_export(temp_xls_file, xdata) + temp_xls_file.seek(0) + workbook = load_workbook(temp_xls_file) + children_sheet = workbook["exp"] + self.assertTrue(children_sheet) + rows = list(children_sheet.rows) + row = [a.value for a in rows[1]] + attachment_id = xdata[0]["_attachments"][0]["id"] + attachment_filename = xdata[0]["_attachments"][0]["filename"] + attachment_url = f"http://example.com/api/v1/files/{attachment_id}?filename={attachment_filename}" # noqa + self.assertIn(attachment_url, row) + + # pylint: disable=invalid-name def test_generation_of_multi_selects_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) - expected_select_multiples =\ - { - 'children': - { - 'children/fav_colors': - [ - 'children/fav_colors/red', 'children/fav_colors/blue', - 'children/fav_colors/pink' - ], - 'children/ice.creams': - [ - 'children/ice.creams/vanilla', - 'children/ice.creams/strawberry', - 'children/ice.creams/chocolate' - ] - } + expected_select_multiples = { + "children": { + "children/fav_colors": [ + "children/fav_colors/red", + "children/fav_colors/blue", + "children/fav_colors/pink", + ], + "children/ice.creams": [ + "children/ice.creams/vanilla", + "children/ice.creams/strawberry", + "children/ice.creams/chocolate", + ], } + } select_multiples = export_builder.select_multiples - self.assertTrue('children' in select_multiples) - self.assertTrue('children/fav_colors' in select_multiples['children']) - self.assertTrue('children/ice.creams' in select_multiples['children']) + self.assertTrue("children" in select_multiples) + self.assertTrue("children/fav_colors" in select_multiples["children"]) + self.assertTrue("children/ice.creams" in select_multiples["children"]) self.assertEqual( - sorted([ - choice['xpath'] for choice in - select_multiples['children']['children/fav_colors']]), sorted( - expected_select_multiples['children']['children/fav_colors'])) + [ + choice["xpath"] + for choice in select_multiples["children"]["children/fav_colors"] + ] + ), + sorted(expected_select_multiples["children"]["children/fav_colors"]), + ) self.assertEqual( - sorted([choice['xpath'] for choice in - select_multiples['children']['children/ice.creams']]), sorted( - expected_select_multiples['children']['children/ice.creams'])) + [ + choice["xpath"] + for choice in select_multiples["children"]["children/ice.creams"] + ] + ), + sorted(expected_select_multiples["children"]["children/ice.creams"]), + ) + # pylint: disable=invalid-name def test_split_select_multiples_works(self): """ Test split_select_multiples works as expected. """ - select_multiples =\ - { - 'children/fav_colors': [ - { - 'xpath': 'children/fav_colors/red', - 'label': 'fav_colors/Red', - }, { - 'xpath': 'children/fav_colors/blue', - 'label': 'fav_colors/Blue', - }, { - 'xpath': 'children/fav_colors/pink', - 'label': 'fav_colors/Pink', - } - ] - } - row = \ - { - 'children/name': 'Mike', - 'children/age': 5, - 'children/fav_colors': 'red blue' - } - new_row = ExportBuilder.split_select_multiples( - row, select_multiples) - expected_row = \ - { - 'children/name': 'Mike', - 'children/age': 5, - 'children/fav_colors': 'red blue', - 'children/fav_colors/red': True, - 'children/fav_colors/blue': True, - 'children/fav_colors/pink': False - } + select_multiples = { + "children/fav_colors": [ + { + "xpath": "children/fav_colors/red", + "label": "fav_colors/Red", + }, + { + "xpath": "children/fav_colors/blue", + "label": "fav_colors/Blue", + }, + { + "xpath": "children/fav_colors/pink", + "label": "fav_colors/Pink", + }, + ] + } + row = { + "children/name": "Mike", + "children/age": 5, + "children/fav_colors": "red blue", + } + new_row = ExportBuilder.split_select_multiples(row, select_multiples) + expected_row = { + "children/name": "Mike", + "children/age": 5, + "children/fav_colors": "red blue", + "children/fav_colors/red": True, + "children/fav_colors/blue": True, + "children/fav_colors/pink": False, + } self.assertEqual(new_row, expected_row) - row = \ - { - 'children/name': 'Mike', - 'children/age': 5, - } - new_row = ExportBuilder.split_select_multiples( - row, select_multiples) - expected_row = \ - { - 'children/name': 'Mike', - 'children/age': 5, - 'children/fav_colors/red': None, - 'children/fav_colors/blue': None, - 'children/fav_colors/pink': None - } + row = { + "children/name": "Mike", + "children/age": 5, + } + new_row = ExportBuilder.split_select_multiples(row, select_multiples) + expected_row = { + "children/name": "Mike", + "children/age": 5, + "children/fav_colors/red": None, + "children/fav_colors/blue": None, + "children/fav_colors/pink": None, + } self.assertEqual(new_row, expected_row) + # pylint: disable=invalid-name def test_split_select_mutliples_works_with_int_value_in_row(self): select_multiples = { - 'children/fav_number': [ + "children/fav_number": [ { - 'xpath': 'children/fav_number/1', - }, { - 'xpath': 'children/fav_number/2', - }, { - 'xpath': 'children/fav_number/3', - } + "xpath": "children/fav_number/1", + }, + { + "xpath": "children/fav_number/2", + }, + { + "xpath": "children/fav_number/3", + }, ] } - row = {'children/fav_number': 1} + row = {"children/fav_number": 1} expected_row = { - 'children/fav_number/1': True, - 'children/fav_number': 1, - 'children/fav_number/3': False, - 'children/fav_number/2': False + "children/fav_number/1": True, + "children/fav_number": 1, + "children/fav_number/3": False, + "children/fav_number/2": False, } new_row = ExportBuilder.split_select_multiples(row, select_multiples) self.assertTrue(new_row) self.assertEqual(new_row, expected_row) + # pylint: disable=invalid-name def test_split_select_multiples_works_when_data_is_blank(self): - select_multiples =\ - { - 'children/fav_colors': [ - { - 'xpath': 'children/fav_colors/red', - 'label': 'fav_colors/Red', - }, { - 'xpath': 'children/fav_colors/blue', - 'label': 'fav_colors/Blue', - }, { - 'xpath': 'children/fav_colors/pink', - 'label': 'fav_colors/Pink', - } - ] - } - row = \ - { - 'children/name': 'Mike', - 'children/age': 5, - 'children/fav_colors': '' - } - new_row = ExportBuilder.split_select_multiples( - row, select_multiples) - expected_row = \ - { - 'children/name': 'Mike', - 'children/age': 5, - 'children/fav_colors': '', - 'children/fav_colors/red': None, - 'children/fav_colors/blue': None, - 'children/fav_colors/pink': None - } + select_multiples = { + "children/fav_colors": [ + { + "xpath": "children/fav_colors/red", + "label": "fav_colors/Red", + }, + { + "xpath": "children/fav_colors/blue", + "label": "fav_colors/Blue", + }, + { + "xpath": "children/fav_colors/pink", + "label": "fav_colors/Pink", + }, + ] + } + row = {"children/name": "Mike", "children/age": 5, "children/fav_colors": ""} + new_row = ExportBuilder.split_select_multiples(row, select_multiples) + expected_row = { + "children/name": "Mike", + "children/age": 5, + "children/fav_colors": "", + "children/fav_colors/red": None, + "children/fav_colors/blue": None, + "children/fav_colors/pink": None, + } self.assertEqual(new_row, expected_row) + # pylint: disable=invalid-name def test_generation_of_gps_fields_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) - expected_gps_fields =\ - { - 'childrens_survey': - { - 'geo/geolocation': - [ - 'geo/_geolocation_latitude', - 'geo/_geolocation_longitude', - 'geo/_geolocation_altitude', - 'geo/_geolocation_precision' - ] - } + expected_gps_fields = { + "childrens_survey": { + "geo/geolocation": [ + "geo/_geolocation_latitude", + "geo/_geolocation_longitude", + "geo/_geolocation_altitude", + "geo/_geolocation_precision", + ] } + } gps_fields = export_builder.gps_fields - self.assertTrue('childrens_survey' in gps_fields) + self.assertTrue("childrens_survey" in gps_fields) self.assertEqual( - sorted(gps_fields['childrens_survey']), - sorted(expected_gps_fields['childrens_survey'])) + sorted(gps_fields["childrens_survey"]), + sorted(expected_gps_fields["childrens_survey"]), + ) def test_split_gps_components_works(self): - gps_fields =\ - { - 'geo/geolocation': - [ - 'geo/_geolocation_latitude', 'geo/_geolocation_longitude', - 'geo/_geolocation_altitude', 'geo/_geolocation_precision' - ] - } - row = \ - { - 'geo/geolocation': '1.0 36.1 2000 20', - } - new_row = ExportBuilder.split_gps_components( - row, gps_fields) - expected_row = \ - { - 'geo/geolocation': '1.0 36.1 2000 20', - 'geo/_geolocation_latitude': '1.0', - 'geo/_geolocation_longitude': '36.1', - 'geo/_geolocation_altitude': '2000', - 'geo/_geolocation_precision': '20' - } + gps_fields = { + "geo/geolocation": [ + "geo/_geolocation_latitude", + "geo/_geolocation_longitude", + "geo/_geolocation_altitude", + "geo/_geolocation_precision", + ] + } + row = { + "geo/geolocation": "1.0 36.1 2000 20", + } + new_row = ExportBuilder.split_gps_components(row, gps_fields) + expected_row = { + "geo/geolocation": "1.0 36.1 2000 20", + "geo/_geolocation_latitude": "1.0", + "geo/_geolocation_longitude": "36.1", + "geo/_geolocation_altitude": "2000", + "geo/_geolocation_precision": "20", + } self.assertEqual(new_row, expected_row) + # pylint: disable=invalid-name def test_split_gps_components_works_when_gps_data_is_blank(self): - gps_fields =\ - { - 'geo/geolocation': - [ - 'geo/_geolocation_latitude', 'geo/_geolocation_longitude', - 'geo/_geolocation_altitude', 'geo/_geolocation_precision' - ] - } - row = \ - { - 'geo/geolocation': '', - } - new_row = ExportBuilder.split_gps_components( - row, gps_fields) - expected_row = \ - { - 'geo/geolocation': '', - } + gps_fields = { + "geo/geolocation": [ + "geo/_geolocation_latitude", + "geo/_geolocation_longitude", + "geo/_geolocation_altitude", + "geo/_geolocation_precision", + ] + } + row = { + "geo/geolocation": "", + } + new_row = ExportBuilder.split_gps_components(row, gps_fields) + expected_row = { + "geo/geolocation": "", + } self.assertEqual(new_row, expected_row) + # pylint: disable=invalid-name def test_generation_of_mongo_encoded_fields_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) - expected_encoded_fields =\ - { - 'childrens_survey': - { - 'tel/tel.office': 'tel/{0}'.format( - _encode_for_mongo('tel.office')), - 'tel/tel.mobile': 'tel/{0}'.format( - _encode_for_mongo('tel.mobile')), - } + expected_encoded_fields = { + "childrens_survey": { + "tel/tel.office": f"tel/{_encode_for_mongo('tel.office')}", + "tel/tel.mobile": f"tel/{_encode_for_mongo('tel.mobile')}", } + } encoded_fields = export_builder.encoded_fields - self.assertTrue('childrens_survey' in encoded_fields) + self.assertTrue("childrens_survey" in encoded_fields) self.assertEqual( - encoded_fields['childrens_survey'], - expected_encoded_fields['childrens_survey']) + encoded_fields["childrens_survey"], + expected_encoded_fields["childrens_survey"], + ) + # pylint: disable=invalid-name def test_decode_fields_names_encoded_for_mongo(self): - encoded_fields = \ - { - 'tel/tel.office': 'tel/{0}'.format( - _encode_for_mongo('tel.office')) - } - row = \ - { - 'name': 'Abe', - 'age': 35, - 'tel/{0}'.format( - _encode_for_mongo('tel.office')): '123-456-789' - } - new_row = ExportBuilder.decode_mongo_encoded_fields( - row, encoded_fields) - expected_row = \ - { - 'name': 'Abe', - 'age': 35, - 'tel/tel.office': '123-456-789' - } + encoded_fields = {"tel/tel.office": f"tel/{_encode_for_mongo('tel.office')}"} + row = { + "name": "Abe", + "age": 35, + f"tel/{_encode_for_mongo('tel.office')}": "123-456-789", + } + new_row = ExportBuilder.decode_mongo_encoded_fields(row, encoded_fields) + expected_row = {"name": "Abe", "age": 35, "tel/tel.office": "123-456-789"} self.assertEqual(new_row, expected_row) def test_generate_field_title(self): self._create_childrens_survey() - field_name = ExportBuilder.format_field_title("children/age", ".", - data_dictionary=self.dd) + field_name = ExportBuilder.format_field_title( + "children/age", ".", data_dictionary=self.data_dictionary + ) expected_field_name = "children.age" self.assertEqual(field_name, expected_field_name) + # pylint: disable=invalid-name def test_delimiter_replacement_works_existing_fields(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = "." export_builder.set_survey(survey) - expected_sections =\ - [ - { - 'name': 'children', - 'elements': [ - { - 'title': 'children.name', - 'xpath': 'children/name' - } - ] - } - ] - children_section = export_builder.section_by_name('children') + expected_sections = [ + { + "name": "children", + "elements": [{"title": "children.name", "xpath": "children/name"}], + } + ] + children_section = export_builder.section_by_name("children") self.assertEqual( - children_section['elements'][0]['title'], - expected_sections[0]['elements'][0]['title']) + children_section["elements"][0]["title"], + expected_sections[0]["elements"][0]["title"], + ) + # pylint: disable=invalid-name def test_delimiter_replacement_works_generated_multi_select_fields(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = "." export_builder.set_survey(survey) - expected_section =\ - { - 'name': 'children', - 'elements': [ - { - 'title': 'children.fav_colors.red', - 'xpath': 'children/fav_colors/red' - } - ] - } - childrens_section = export_builder.section_by_name('children') - match = [x for x in childrens_section['elements'] - if expected_section['elements'][0]['xpath'] == x['xpath']][0] - self.assertEqual( - expected_section['elements'][0]['title'], match['title']) + expected_section = { + "name": "children", + "elements": [ + {"title": "children.fav_colors.red", "xpath": "children/fav_colors/red"} + ], + } + childrens_section = export_builder.section_by_name("children") + match = [ + x + for x in childrens_section["elements"] + if expected_section["elements"][0]["xpath"] == x["xpath"] + ][0] + self.assertEqual(expected_section["elements"][0]["title"], match["title"]) + # pylint: disable=invalid-name def test_delimiter_replacement_works_for_generated_gps_fields(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = "." export_builder.set_survey(survey) - expected_section = \ - { - 'name': 'childrens_survey', - 'elements': [ - { - 'title': 'geo._geolocation_latitude', - 'xpath': 'geo/_geolocation_latitude' - } - ] - } - main_section = export_builder.section_by_name('childrens_survey') - match = [x for x in main_section['elements'] - if expected_section['elements'][0]['xpath'] == x['xpath']][0] - self.assertEqual( - expected_section['elements'][0]['title'], match['title']) + expected_section = { + "name": "childrens_survey", + "elements": [ + { + "title": "geo._geolocation_latitude", + "xpath": "geo/_geolocation_latitude", + } + ], + } + main_section = export_builder.section_by_name("childrens_survey") + match = [ + x + for x in main_section["elements"] + if expected_section["elements"][0]["xpath"] == x["xpath"] + ][0] + self.assertEqual(expected_section["elements"][0]["title"], match["title"]) def test_to_xls_export_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) - xls_file = NamedTemporaryFile(suffix='.xlsx') - filename = xls_file.name - export_builder.to_xls_export(filename, self.data) - xls_file.seek(0) - wb = load_workbook(filename) - # check that we have childrens_survey, children, children_cartoons - # and children_cartoons_characters sheets - expected_sheet_names = ['childrens_survey', 'children', - 'children_cartoons', - 'children_cartoons_characters'] - self.assertEqual(wb.get_sheet_names(), expected_sheet_names) - # check header columns - main_sheet = wb.get_sheet_by_name('childrens_survey') - expected_column_headers = ( - 'name', 'age', 'geo/geolocation', 'geo/_geolocation_latitude', - 'geo/_geolocation_longitude', 'geo/_geolocation_altitude', - 'geo/_geolocation_precision', 'tel/tel.office', - 'tel/tel.mobile', '_id', 'meta/instanceID', '_uuid', - '_submission_time', '_index', '_parent_index', - '_parent_table_name', '_tags', '_notes', '_version', - '_duration', '_submitted_by') - - column_headers = tuple(main_sheet.values)[0] - self.assertEqual(sorted(column_headers), - sorted(expected_column_headers)) - - childrens_sheet = wb.get_sheet_by_name('children') - expected_column_headers = ( - 'children/name', 'children/age', 'children/fav_colors', - 'children/fav_colors/red', 'children/fav_colors/blue', - 'children/fav_colors/pink', 'children/ice.creams', - 'children/ice.creams/vanilla', 'children/ice.creams/strawberry', - 'children/ice.creams/chocolate', '_id', '_uuid', - '_submission_time', '_index', '_parent_index', - '_parent_table_name', '_tags', '_notes', '_version', - '_duration', '_submitted_by') - column_headers = tuple(childrens_sheet.values)[0] - self.assertEqual(sorted(column_headers), - sorted(expected_column_headers)) - - cartoons_sheet = wb.get_sheet_by_name('children_cartoons') - expected_column_headers = ( - 'children/cartoons/name', 'children/cartoons/why', '_id', - '_uuid', '_submission_time', '_index', '_parent_index', - '_parent_table_name', '_tags', '_notes', '_version', - '_duration', '_submitted_by') - column_headers = tuple(cartoons_sheet.values)[0] - self.assertEqual(sorted(column_headers), - sorted(expected_column_headers)) - - characters_sheet = wb.get_sheet_by_name('children_cartoons_characters') - expected_column_headers = ( - 'children/cartoons/characters/name', - 'children/cartoons/characters/good_or_evil', '_id', '_uuid', - '_submission_time', '_index', '_parent_index', - '_parent_table_name', '_tags', '_notes', '_version', - '_duration', '_submitted_by') - column_headers = tuple(characters_sheet.values)[0] - self.assertEqual(sorted(column_headers), - sorted(expected_column_headers)) - - xls_file.close() + with NamedTemporaryFile(suffix=".xlsx") as xls_file: + filename = xls_file.name + export_builder.to_xls_export(filename, self.data) + xls_file.seek(0) + workbook = load_workbook(filename) + # check that we have childrens_survey, children, children_cartoons + # and children_cartoons_characters sheets + expected_sheet_names = [ + "childrens_survey", + "children", + "children_cartoons", + "children_cartoons_characters", + ] + self.assertEqual(list(workbook.get_sheet_names()), expected_sheet_names) + # check header columns + main_sheet = workbook.get_sheet_by_name("childrens_survey") + expected_column_headers = [ + "name", + "age", + "geo/geolocation", + "geo/_geolocation_latitude", + "geo/_geolocation_longitude", + "geo/_geolocation_altitude", + "geo/_geolocation_precision", + "tel/tel.office", + "tel/tel.mobile", + "_id", + "meta/instanceID", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + column_headers = list(main_sheet.values)[0] + self.assertEqual( + sorted(list(column_headers)), sorted(expected_column_headers) + ) + + childrens_sheet = workbook.get_sheet_by_name("children") + expected_column_headers = [ + "children/name", + "children/age", + "children/fav_colors", + "children/fav_colors/red", + "children/fav_colors/blue", + "children/fav_colors/pink", + "children/ice.creams", + "children/ice.creams/vanilla", + "children/ice.creams/strawberry", + "children/ice.creams/chocolate", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + column_headers = list(childrens_sheet.values)[0] + self.assertEqual( + sorted(list(column_headers)), sorted(expected_column_headers) + ) + + cartoons_sheet = workbook.get_sheet_by_name("children_cartoons") + expected_column_headers = [ + "children/cartoons/name", + "children/cartoons/why", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + column_headers = list(cartoons_sheet.values)[0] + self.assertEqual( + sorted(list(column_headers)), sorted(expected_column_headers) + ) + + characters_sheet = workbook.get_sheet_by_name( + "children_cartoons_characters" + ) + expected_column_headers = [ + "children/cartoons/characters/name", + "children/cartoons/characters/good_or_evil", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + column_headers = list(characters_sheet.values)[0] + self.assertEqual( + sorted(list(column_headers)), sorted(expected_column_headers) + ) + + # pylint: disable=invalid-name def test_to_xls_export_respects_custom_field_delimiter(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = ExportBuilder.GROUP_DELIMITER_DOT export_builder.set_survey(survey) - xls_file = NamedTemporaryFile(suffix='.xlsx') - filename = xls_file.name - export_builder.to_xls_export(filename, self.data) - xls_file.seek(0) - wb = load_workbook(filename) - - # check header columns - main_sheet = wb.get_sheet_by_name('childrens_survey') - expected_column_headers = ( - 'name', 'age', 'geo.geolocation', 'geo._geolocation_latitude', - 'geo._geolocation_longitude', 'geo._geolocation_altitude', - 'geo._geolocation_precision', 'tel.tel.office', - 'tel.tel.mobile', '_id', 'meta.instanceID', '_uuid', - '_submission_time', '_index', '_parent_index', - '_parent_table_name', '_tags', '_notes', '_version', - '_duration', '_submitted_by') - column_headers = tuple(main_sheet.values)[0] - self.assertEqual(sorted(column_headers), - sorted(expected_column_headers)) - xls_file.close() + with NamedTemporaryFile(suffix=".xlsx") as xls_file: + filename = xls_file.name + export_builder.to_xls_export(filename, self.data) + xls_file.seek(0) + workbook = load_workbook(filename) + + # check header columns + main_sheet = workbook.get_sheet_by_name("childrens_survey") + expected_column_headers = [ + "name", + "age", + "geo.geolocation", + "geo._geolocation_latitude", + "geo._geolocation_longitude", + "geo._geolocation_altitude", + "geo._geolocation_precision", + "tel.tel.office", + "tel.tel.mobile", + "_id", + "meta.instanceID", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + column_headers = list(main_sheet.values)[0] + self.assertEqual( + sorted(list(column_headers)), sorted(expected_column_headers) + ) + # pylint: disable=invalid-name def test_get_valid_sheet_name_catches_duplicates(self): - work_sheets = {'childrens_survey': "Worksheet"} + work_sheets = {"childrens_survey": "Worksheet"} desired_sheet_name = "childrens_survey" expected_sheet_name = "childrens_survey1" generated_sheet_name = ExportBuilder.get_valid_sheet_name( - desired_sheet_name, work_sheets) + desired_sheet_name, work_sheets + ) self.assertEqual(generated_sheet_name, expected_sheet_name) + # pylint: disable=invalid-name def test_get_valid_sheet_name_catches_long_names(self): desired_sheet_name = "childrens_survey_with_a_very_long_name" expected_sheet_name = "childrens_survey_with_a_very_lo" generated_sheet_name = ExportBuilder.get_valid_sheet_name( - desired_sheet_name, []) + desired_sheet_name, [] + ) self.assertEqual(generated_sheet_name, expected_sheet_name) + # pylint: disable=invalid-name def test_get_valid_sheet_name_catches_long_duplicate_names(self): - work_sheet_titles = ['childrens_survey_with_a_very_lo'] + work_sheet_titles = ["childrens_survey_with_a_very_lo"] desired_sheet_name = "childrens_survey_with_a_very_long_name" expected_sheet_name = "childrens_survey_with_a_very_l1" generated_sheet_name = ExportBuilder.get_valid_sheet_name( - desired_sheet_name, work_sheet_titles) + desired_sheet_name, work_sheet_titles + ) self.assertEqual(generated_sheet_name, expected_sheet_name) + # pylint: disable=invalid-name def test_to_xls_export_generates_valid_sheet_names(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_with_a_very_long_name.xlsx'), - default_name='childrens_survey_with_a_very_long_name') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_with_a_very_long_name.xlsx"), + default_name="childrens_survey_with_a_very_long_name", + ) export_builder = ExportBuilder() export_builder.set_survey(survey) - xls_file = NamedTemporaryFile(suffix='.xlsx') - filename = xls_file.name - export_builder.to_xls_export(filename, self.data) - xls_file.seek(0) - wb = load_workbook(filename) - # check that we have childrens_survey, children, children_cartoons - # and children_cartoons_characters sheets - expected_sheet_names = ['childrens_survey_with_a_very_lo', - 'childrens_survey_with_a_very_l1', - 'childrens_survey_with_a_very_l2', - 'childrens_survey_with_a_very_l3'] - self.assertEqual(wb.get_sheet_names(), expected_sheet_names) - xls_file.close() + with NamedTemporaryFile(suffix=".xlsx") as xls_file: + filename = xls_file.name + export_builder.to_xls_export(filename, self.data) + xls_file.seek(0) + workbook = load_workbook(filename) + # check that we have childrens_survey, children, children_cartoons + # and children_cartoons_characters sheets + expected_sheet_names = [ + "childrens_survey_with_a_very_lo", + "childrens_survey_with_a_very_l1", + "childrens_survey_with_a_very_l2", + "childrens_survey_with_a_very_l3", + ] + self.assertEqual(list(workbook.get_sheet_names()), expected_sheet_names) + # pylint: disable=invalid-name def test_child_record_parent_table_is_updated_when_sheet_is_renamed(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_with_a_very_long_name.xlsx'), - default_name='childrens_survey_with_a_very_long_name') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_with_a_very_long_name.xlsx"), + default_name="childrens_survey_with_a_very_long_name", + ) export_builder = ExportBuilder() export_builder.set_survey(survey) - xls_file = NamedTemporaryFile(suffix='.xlsx') - filename = xls_file.name - export_builder.to_xls_export(filename, self.long_survey_data) - xls_file.seek(0) - wb = load_workbook(filename) - - # get the children's sheet - ws1 = wb['childrens_survey_with_a_very_l1'] - - # parent_table is in cell K2 - parent_table_name = ws1['K2'].value - expected_parent_table_name = 'childrens_survey_with_a_very_lo' - self.assertEqual(parent_table_name, expected_parent_table_name) - - # get cartoons sheet - ws2 = wb['childrens_survey_with_a_very_l2'] - parent_table_name = ws2['G2'].value - expected_parent_table_name = 'childrens_survey_with_a_very_l1' - self.assertEqual(parent_table_name, expected_parent_table_name) - xls_file.close() + with NamedTemporaryFile(suffix=".xlsx") as xls_file: + filename = xls_file.name + export_builder.to_xls_export(filename, self.long_survey_data) + xls_file.seek(0) + workbook = load_workbook(filename) + + # get the children's sheet + ws1 = workbook["childrens_survey_with_a_very_l1"] + + # parent_table is in cell K2 + parent_table_name = ws1["K2"].value + expected_parent_table_name = "childrens_survey_with_a_very_lo" + self.assertEqual(parent_table_name, expected_parent_table_name) + + # get cartoons sheet + ws2 = workbook["childrens_survey_with_a_very_l2"] + parent_table_name = ws2["G2"].value + expected_parent_table_name = "childrens_survey_with_a_very_l1" + self.assertEqual(parent_table_name, expected_parent_table_name) def test_type_conversion(self): submission_1 = { @@ -1747,15 +1867,12 @@ def test_type_conversion(self): "_uuid": "2a8129f5-3091-44e1-a579-bed2b07a12cf", "when": "2013-07-03", "amount": "250.0", - "_geolocation": [ - "-1.2625482", - "36.7924794" - ], + "_geolocation": ["-1.2625482", "36.7924794"], "_xform_id_string": "test_data_types", "_userform_id": "larryweya_test_data_types", "_status": "submitted_via_web", "precisely": "2013-07-03T15:24:00.000+03", - "really": "15:24:00.000+03" + "really": "15:24:00.000+03", } submission_2 = { @@ -1771,82 +1888,87 @@ def test_type_conversion(self): "amount": "", } - survey = create_survey_from_xls(viewer_fixture_path( - 'test_data_types/test_data_types.xlsx'), - default_name='test_data_types') + survey = create_survey_from_xls( + viewer_fixture_path("test_data_types/test_data_types.xlsx"), + default_name="test_data_types", + ) export_builder = ExportBuilder() export_builder.set_survey(survey) # format submission 1 for export survey_name = survey.name indices = {survey_name: 0} - data = dict_to_joined_export(submission_1, 1, indices, survey_name, - survey, submission_1) - new_row = export_builder.pre_process_row(data[survey_name], - export_builder.sections[0]) - self.assertIsInstance(new_row['age'], int) - self.assertIsInstance(new_row['when'], datetime.date) - self.assertIsInstance(new_row['amount'], float) + data = dict_to_joined_export( + submission_1, 1, indices, survey_name, survey, submission_1 + ) + new_row = export_builder.pre_process_row( + data[survey_name], export_builder.sections[0] + ) + self.assertIsInstance(new_row["age"], int) + self.assertIsInstance(new_row["when"], datetime.date) + self.assertIsInstance(new_row["amount"], float) # check missing values dont break and empty values return blank strings indices = {survey_name: 0} - data = dict_to_joined_export(submission_2, 1, indices, survey_name, - survey, submission_2) - new_row = export_builder.pre_process_row(data[survey_name], - export_builder.sections[0]) - self.assertIsInstance(new_row['amount'], basestring) - self.assertEqual(new_row['amount'], '') + data = dict_to_joined_export( + submission_2, 1, indices, survey_name, survey, submission_2 + ) + new_row = export_builder.pre_process_row( + data[survey_name], export_builder.sections[0] + ) + self.assertIsInstance(new_row["amount"], str) + self.assertEqual(new_row["amount"], "") + # pylint: disable=invalid-name def test_xls_convert_dates_before_1900(self): - survey = create_survey_from_xls(viewer_fixture_path( - 'test_data_types/test_data_types.xlsx'), - default_name='test_data_types') + survey = create_survey_from_xls( + viewer_fixture_path("test_data_types/test_data_types.xlsx"), + default_name="test_data_types", + ) export_builder = ExportBuilder() export_builder.set_survey(survey) data = [ { - 'name': 'Abe', - 'when': '1899-07-03', + "name": "Abe", + "when": "1899-07-03", } ] # create export file - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - export_builder.to_xls_export(temp_xls_file.name, data) - temp_xls_file.close() + with NamedTemporaryFile(suffix=".xlsx") as temp_xls_file: + export_builder.to_xls_export(temp_xls_file.name, data) # this should error if there is a problem, not sure what to assert def test_convert_types(self): - val = '1' + val = "1" expected_val = 1 - converted_val = ExportBuilder.convert_type(val, 'int') + converted_val = ExportBuilder.convert_type(val, "int") self.assertIsInstance(converted_val, int) self.assertEqual(converted_val, expected_val) - val = '1.2' + val = "1.2" expected_val = 1.2 - converted_val = ExportBuilder.convert_type(val, 'decimal') + converted_val = ExportBuilder.convert_type(val, "decimal") self.assertIsInstance(converted_val, float) self.assertEqual(converted_val, expected_val) - val = '2012-06-23' + val = "2012-06-23" expected_val = datetime.date(2012, 6, 23) - converted_val = ExportBuilder.convert_type(val, 'date') + converted_val = ExportBuilder.convert_type(val, "date") self.assertIsInstance(converted_val, datetime.date) self.assertEqual(converted_val, expected_val) + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_to_sav_export(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - filename = temp_zip_file.name - export_builder.to_zipped_sav(filename, self.data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + filename = temp_zip_file.name + export_builder.to_zipped_sav(filename, self.data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # generate data to compare with index = 1 @@ -1855,55 +1977,52 @@ def test_to_sav_export(self): outputs = [] for d in self.data: outputs.append( - dict_to_joined_export( - d, index, indices, survey_name, survey, d)) + dict_to_joined_export(d, index, indices, survey_name, survey, d) + ) index += 1 # check that each file exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "{0}.sav".format(survey.name)))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, f"{survey.name}.sav"))) def _test_sav_file(section): with SavReader( - os.path.join( - temp_dir, "{0}.sav".format(section)), - returnHeader=True) as reader: + os.path.join(temp_dir, f"{section}.sav"), returnHeader=True + ) as reader: header = next(reader) - rows = [r for r in reader] + rows = list(reader) # open comparison file - with SavReader(_logger_fixture_path( - 'spss', "{0}.sav".format(section)), - returnHeader=True) as fixture_reader: + with SavReader( + _logger_fixture_path("spss", f"{section}.sav"), + returnHeader=True, + ) as fixture_reader: fixture_header = next(fixture_reader) self.assertEqual(header, fixture_header) - expected_rows = [r for r in fixture_reader] + expected_rows = list(fixture_reader) self.assertEqual(rows, expected_rows) - if section == 'children_cartoons_charactors': - self.assertEqual(reader.valueLabels, { - 'good_or_evil': {'good': 'Good'} - }) + if section == "children_cartoons_charactors": + self.assertEqual( + reader.valueLabels, {"good_or_evil": {"good": "Good"}} + ) for section in export_builder.sections: - section_name = section['name'].replace('/', '_') + section_name = section["name"].replace("/", "_") _test_sav_file(section_name) + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_to_sav_export_language(self): - survey = self._create_childrens_survey('childrens_survey_sw.xlsx') + survey = self._create_childrens_survey("childrens_survey_sw.xlsx") export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - filename = temp_zip_file.name - export_builder.to_zipped_sav(filename, self.data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + filename = temp_zip_file.name + export_builder.to_zipped_sav(filename, self.data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # generate data to compare with index = 1 @@ -1912,301 +2031,339 @@ def test_to_sav_export_language(self): outputs = [] for d in self.data: outputs.append( - dict_to_joined_export( - d, index, indices, survey_name, survey, d)) + dict_to_joined_export(d, index, indices, survey_name, survey, d) + ) index += 1 # check that each file exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "{0}.sav".format(survey.name)))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, f"{survey.name}.sav"))) def _test_sav_file(section): with SavReader( - os.path.join( - temp_dir, "{0}.sav".format(section)), - returnHeader=True) as reader: + os.path.join(temp_dir, f"{section}.sav"), returnHeader=True + ) as reader: header = next(reader) - rows = [r for r in reader] - if section != 'childrens_survey_sw': - section += '_sw' + rows = list(reader) + if section != "childrens_survey_sw": + section += "_sw" # open comparison file - with SavReader(_logger_fixture_path( - 'spss', "{0}.sav".format(section)), - returnHeader=True) as fixture_reader: + with SavReader( + _logger_fixture_path("spss", f"{section}.sav"), + returnHeader=True, + ) as fixture_reader: fixture_header = next(fixture_reader) self.assertEqual(header, fixture_header) - expected_rows = [r for r in fixture_reader] + expected_rows = list(fixture_reader) self.assertEqual(rows, expected_rows) - if section == 'children_cartoons_charactors': - self.assertEqual(reader.valueLabels, { - 'good_or_evil': {'good': 'Good'} - }) + if section == "children_cartoons_charactors": + self.assertEqual( + reader.valueLabels, {"good_or_evil": {"good": "Good"}} + ) for section in export_builder.sections: - section_name = section['name'].replace('/', '_') + section_name = section["name"].replace("/", "_") _test_sav_file(section_name) + # pylint: disable=invalid-name def test_generate_field_title_truncated_titles(self): self._create_childrens_survey() - field_name = ExportBuilder.format_field_title("children/age", "/", - data_dictionary=self.dd, - remove_group_name=True) + field_name = ExportBuilder.format_field_title( + "children/age", + "/", + data_dictionary=self.data_dictionary, + remove_group_name=True, + ) expected_field_name = "age" self.assertEqual(field_name, expected_field_name) + # pylint: disable=invalid-name def test_generate_field_title_truncated_titles_select_multiple(self): self._create_childrens_survey() field_name = ExportBuilder.format_field_title( - "children/fav_colors/red", "/", - data_dictionary=self.dd, - remove_group_name=True + "children/fav_colors/red", + "/", + data_dictionary=self.data_dictionary, + remove_group_name=True, ) expected_field_name = "fav_colors/red" self.assertEqual(field_name, expected_field_name) + # pylint: disable=invalid-name def test_xls_export_remove_group_name(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xlsx'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xlsx"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) - temp_xls_file.seek(0) - # check that values for red\'s and blue\'s are set to true - wb = load_workbook(temp_xls_file.name) - children_sheet = wb["children.info"] - data = dict([(r[0].value, r[1].value) for r in children_sheet.columns]) - self.assertTrue(data[u"fav_colors/red's"]) - self.assertTrue(data[u"fav_colors/blue's"]) - self.assertFalse(data[u"fav_colors/pink's"]) - temp_xls_file.close() + with NamedTemporaryFile(suffix=".xlsx") as temp_xls_file: + export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) + temp_xls_file.seek(0) + # check that values for red\'s and blue\'s are set to true + workbook = load_workbook(temp_xls_file.name) + children_sheet = workbook["children.info"] + data = {r[0].value: r[1].value for r in children_sheet.columns} + self.assertTrue(data["fav_colors/red's"]) + self.assertTrue(data["fav_colors/blue's"]) + self.assertFalse(data["fav_colors/pink's"]) def test_zipped_csv_export_remove_group_name(self): """ cvs writer doesnt handle unicode we we have to encode to ascii """ - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xlsx'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xlsx"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children.info.csv"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.info.csv"))) # check file's contents - with open(os.path.join(temp_dir, "children.info.csv")) as csv_file: + with open( + os.path.join(temp_dir, "children.info.csv"), encoding="utf-8" + ) as csv_file: reader = csv.reader(csv_file) - expected_headers = ['name.first', - 'age', - 'fav_colors', - 'fav_colors/red\'s', - 'fav_colors/blue\'s', - 'fav_colors/pink\'s', - 'ice_creams', - 'ice_creams/vanilla', - 'ice_creams/strawberry', - 'ice_creams/chocolate', '_id', - '_uuid', '_submission_time', '_index', - '_parent_table_name', '_parent_index', - '_tags', '_notes', '_version', - '_duration', '_submitted_by'] - rows = [row for row in reader] - actual_headers = [h for h in rows[0]] + expected_headers = [ + "name.first", + "age", + "fav_colors", + "fav_colors/red's", + "fav_colors/blue's", + "fav_colors/pink's", + "ice_creams", + "ice_creams/vanilla", + "ice_creams/strawberry", + "ice_creams/chocolate", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + rows = list(reader) + actual_headers = list(rows[0]) self.assertEqual(sorted(actual_headers), sorted(expected_headers)) data = dict(zip(rows[0], rows[1])) - self.assertEqual(data['fav_colors/red\'s'], 'True') - self.assertEqual(data['fav_colors/blue\'s'], 'True') - self.assertEqual(data['fav_colors/pink\'s'], 'False') + self.assertEqual(data["fav_colors/red's"], "True") + self.assertEqual(data["fav_colors/blue's"], "True") + self.assertEqual(data["fav_colors/pink's"], "False") # check that red and blue are set to true shutil.rmtree(temp_dir) def test_xls_export_with_labels(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xlsx'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xlsx"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) - temp_xls_file.seek(0) - # check that values for red\'s and blue\'s are set to true - wb = load_workbook(temp_xls_file.name) - children_sheet = wb["children.info"] - labels = dict([(r[0].value, r[1].value) - for r in children_sheet.columns]) - self.assertEqual(labels['name.first'], '3.1 Childs name') - self.assertEqual(labels['age'], '3.2 Child age') - self.assertEqual(labels['fav_colors/red\'s'], 'fav_colors/Red') - self.assertEqual(labels['fav_colors/blue\'s'], 'fav_colors/Blue') - self.assertEqual(labels['fav_colors/pink\'s'], 'fav_colors/Pink') - - data = dict([(r[0].value, r[2].value) for r in children_sheet.columns]) - self.assertEqual(data['name.first'], 'Mike') - self.assertEqual(data['age'], 5) - self.assertTrue(data['fav_colors/red\'s']) - self.assertTrue(data['fav_colors/blue\'s']) - self.assertFalse(data['fav_colors/pink\'s']) - temp_xls_file.close() - + with NamedTemporaryFile(suffix=".xlsx") as temp_xls_file: + export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) + temp_xls_file.seek(0) + # check that values for red\'s and blue\'s are set to true + workbook = load_workbook(temp_xls_file.name) + children_sheet = workbook["children.info"] + labels = {r[0].value: r[1].value for r in children_sheet.columns} + self.assertEqual(labels["name.first"], "3.1 Childs name") + self.assertEqual(labels["age"], "3.2 Child age") + self.assertEqual(labels["fav_colors/red's"], "fav_colors/Red") + self.assertEqual(labels["fav_colors/blue's"], "fav_colors/Blue") + self.assertEqual(labels["fav_colors/pink's"], "fav_colors/Pink") + + data = {r[0].value: r[2].value for r in children_sheet.columns} + self.assertEqual(data["name.first"], "Mike") + self.assertEqual(data["age"], 5) + self.assertTrue(data["fav_colors/red's"]) + self.assertTrue(data["fav_colors/blue's"]) + self.assertFalse(data["fav_colors/pink's"]) + + # pylint: disable=invalid-name def test_xls_export_with_labels_only(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xlsx'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xlsx"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS_ONLY = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) temp_xls_file.seek(0) # check that values for red\'s and blue\'s are set to true wb = load_workbook(temp_xls_file.name) children_sheet = wb["children.info"] - data = dict([(r[0].value, r[1].value) for r in children_sheet.columns]) - self.assertEqual(data['3.1 Childs name'], 'Mike') - self.assertEqual(data['3.2 Child age'], 5) - self.assertTrue(data['fav_colors/Red']) - self.assertTrue(data['fav_colors/Blue']) - self.assertFalse(data['fav_colors/Pink']) + data = {r[0].value: r[1].value for r in children_sheet.columns} + self.assertEqual(data["3.1 Childs name"], "Mike") + self.assertEqual(data["3.2 Child age"], 5) + self.assertTrue(data["fav_colors/Red"]) + self.assertTrue(data["fav_colors/Blue"]) + self.assertFalse(data["fav_colors/Pink"]) temp_xls_file.close() + # pylint: disable=invalid-name def test_zipped_csv_export_with_labels(self): """ cvs writer doesnt handle unicode we we have to encode to ascii """ - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xlsx'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xlsx"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children.info.csv"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.info.csv"))) # check file's contents - with open(os.path.join(temp_dir, "children.info.csv"), - encoding='utf-8') as csv_file: + with open( + os.path.join(temp_dir, "children.info.csv"), encoding="utf-8" + ) as csv_file: reader = csv.reader(csv_file) - expected_headers = ['name.first', - 'age', - 'fav_colors', - 'fav_colors/red\'s', - 'fav_colors/blue\'s', - 'fav_colors/pink\'s', - 'ice_creams', - 'ice_creams/vanilla', - 'ice_creams/strawberry', - 'ice_creams/chocolate', '_id', - '_uuid', '_submission_time', '_index', - '_parent_table_name', '_parent_index', - '_tags', '_notes', '_version', - '_duration', '_submitted_by'] - expected_labels = ['3.1 Childs name', - '3.2 Child age', - '3.3 Favorite Colors', - 'fav_colors/Red', - 'fav_colors/Blue', - 'fav_colors/Pink', - '3.3 Ice Creams', - 'ice_creams/Vanilla', - 'ice_creams/Strawberry', - 'ice_creams/Chocolate', '_id', - '_uuid', '_submission_time', '_index', - '_parent_table_name', '_parent_index', - '_tags', '_notes', '_version', - '_duration', '_submitted_by'] - rows = [row for row in reader] - actual_headers = [h for h in rows[0]] + expected_headers = [ + "name.first", + "age", + "fav_colors", + "fav_colors/red's", + "fav_colors/blue's", + "fav_colors/pink's", + "ice_creams", + "ice_creams/vanilla", + "ice_creams/strawberry", + "ice_creams/chocolate", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + expected_labels = [ + "3.1 Childs name", + "3.2 Child age", + "3.3 Favorite Colors", + "fav_colors/Red", + "fav_colors/Blue", + "fav_colors/Pink", + "3.3 Ice Creams", + "ice_creams/Vanilla", + "ice_creams/Strawberry", + "ice_creams/Chocolate", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + rows = list(reader) + actual_headers = list(rows[0]) self.assertEqual(sorted(actual_headers), sorted(expected_headers)) - actual_labels = [h for h in rows[1]] + actual_labels = list(rows[1]) self.assertEqual(sorted(actual_labels), sorted(expected_labels)) data = dict(zip(rows[0], rows[2])) - self.assertEqual(data['fav_colors/red\'s'], 'True') - self.assertEqual(data['fav_colors/blue\'s'], 'True') - self.assertEqual(data['fav_colors/pink\'s'], 'False') + self.assertEqual(data["fav_colors/red's"], "True") + self.assertEqual(data["fav_colors/blue's"], "True") + self.assertEqual(data["fav_colors/pink's"], "False") # check that red and blue are set to true shutil.rmtree(temp_dir) + # pylint: disable=invalid-name def test_zipped_csv_export_with_labels_only(self): """ - cvs writer doesnt handle unicode we we have to encode to ascii + csv writer does not handle unicode we we have to encode to ascii """ - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xlsx'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xlsx"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS_ONLY = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() - # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children.info.csv"))) + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) + # check that the children's file (which has the unicode header) exists + self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.info.csv"))) # check file's contents - with open(os.path.join(temp_dir, "children.info.csv"), - encoding='utf-8') as csv_file: + with open( + os.path.join(temp_dir, "children.info.csv"), encoding="utf-8" + ) as csv_file: reader = csv.reader(csv_file) expected_headers = [ - '3.1 Childs name', - '3.2 Child age', - '3.3 Favorite Colors', - 'fav_colors/Red', - 'fav_colors/Blue', - 'fav_colors/Pink', - '3.3 Ice Creams', - 'ice_creams/Vanilla', - 'ice_creams/Strawberry', - 'ice_creams/Chocolate', '_id', - '_uuid', '_submission_time', '_index', - '_parent_table_name', '_parent_index', - '_tags', '_notes', '_version', - '_duration', '_submitted_by' + "3.1 Childs name", + "3.2 Child age", + "3.3 Favorite Colors", + "fav_colors/Red", + "fav_colors/Blue", + "fav_colors/Pink", + "3.3 Ice Creams", + "ice_creams/Vanilla", + "ice_creams/Strawberry", + "ice_creams/Chocolate", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", ] - rows = [row for row in reader] - actual_headers = [h for h in rows[0]] + rows = list(reader) + actual_headers = list(rows[0]) self.assertEqual(sorted(actual_headers), sorted(expected_headers)) data = dict(zip(rows[0], rows[1])) - self.assertEqual(data['fav_colors/Red'], 'True') - self.assertEqual(data['fav_colors/Blue'], 'True') - self.assertEqual(data['fav_colors/Pink'], 'False') + self.assertEqual(data["fav_colors/Red"], "True") + self.assertEqual(data["fav_colors/Blue"], "True") + self.assertEqual(data["fav_colors/Pink"], "False") # check that red and blue are set to true shutil.rmtree(temp_dir) + @skipIf(SavHeaderReader is None, "savReaderWriter is not supported now.") def test_to_sav_export_with_labels(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() @@ -2214,279 +2371,296 @@ def test_to_sav_export_with_labels(self): export_builder.set_survey(survey) export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - filename = temp_zip_file.name - export_builder.to_zipped_sav(filename, self.data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + filename = temp_zip_file.name + export_builder.to_zipped_sav(filename, self.data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # generate data to compare with index = 1 indices = {} survey_name = survey.name outputs = [] - for d in self.data: + for item in self.data: outputs.append( - dict_to_joined_export( - d, index, indices, survey_name, survey, d)) + dict_to_joined_export(item, index, indices, survey_name, survey, item) + ) index += 1 # check that each file exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "{0}.sav".format(survey.name)))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, f"{survey.name}.sav"))) def _test_sav_file(section): - sav_path = os.path.join(temp_dir, "{0}.sav".format(section)) - if section == 'children_survey': + sav_path = os.path.join(temp_dir, f"{section}.sav") + if section == "children_survey": with SavHeaderReader(sav_path) as header: expected_labels = [ - '1. What is your name?', '2. How old are you?', - '4. Geo-location', '5.1 Office telephone', - '5.2 Mobile telephone', '_duration', '_id', - '_index', '_notes', '_parent_index', - '_parent_table_name', '_submission_time', - '_submitted_by', - '_tags', '_uuid', '_version', - 'geo/_geolocation_altitude', - 'geo/_geolocation_latitude', - 'geo/_geolocation_longitude', - 'geo/_geolocation_precision', - 'meta/instanceID' + "1. What is your name?", + "2. How old are you?", + "4. Geo-location", + "5.1 Office telephone", + "5.2 Mobile telephone", + "_duration", + "_id", + "_index", + "_notes", + "_parent_index", + "_parent_table_name", + "_submission_time", + "_submitted_by", + "_tags", + "_uuid", + "_version", + "geo/_geolocation_altitude", + "geo/_geolocation_latitude", + "geo/_geolocation_longitude", + "geo/_geolocation_precision", + "meta/instanceID", ] labels = header.varLabels.values() self.assertEqual(sorted(expected_labels), sorted(labels)) with SavReader(sav_path, returnHeader=True) as reader: header = next(reader) - rows = [r for r in reader] + rows = list(reader) # open comparison file - with SavReader(_logger_fixture_path( - 'spss', "{0}.sav".format(section)), - returnHeader=True) as fixture_reader: + with SavReader( + _logger_fixture_path("spss", f"{section}.sav"), + returnHeader=True, + ) as fixture_reader: fixture_header = next(fixture_reader) self.assertEqual(header, fixture_header) - expected_rows = [r for r in fixture_reader] + expected_rows = list(fixture_reader) self.assertEqual(rows, expected_rows) for section in export_builder.sections: - section_name = section['name'].replace('/', '_') + section_name = section["name"].replace("/", "_") _test_sav_file(section_name) + # pylint: disable=invalid-name def test_xls_export_with_english_labels(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_en.xlsx'), - default_name='childrens_survey_en') - # no default_language is not set - self.assertEqual( - survey.to_json_dict().get('default_language'), 'default' + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_en.xlsx"), + default_name="childrens_survey_en", ) + # no default_language is not set + self.assertEqual(survey.to_json_dict().get("default_language"), "default") export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - export_builder.to_xls_export(temp_xls_file.name, self.data) - temp_xls_file.seek(0) - wb = load_workbook(temp_xls_file.name) - childrens_survey_sheet = wb["childrens_survey_en"] - labels = dict([(r[0].value, r[1].value) - for r in childrens_survey_sheet.columns]) - self.assertEqual(labels['name'], '1. What is your name?') - self.assertEqual(labels['age'], '2. How old are you?') - - children_sheet = wb["children"] - labels = dict([(r[0].value, r[1].value) - for r in children_sheet.columns]) - self.assertEqual(labels['fav_colors/red'], 'fav_colors/Red') - self.assertEqual(labels['fav_colors/blue'], 'fav_colors/Blue') - temp_xls_file.close() - + with NamedTemporaryFile(suffix=".xlsx") as temp_xls_file: + export_builder.to_xls_export(temp_xls_file.name, self.data) + temp_xls_file.seek(0) + workbook = load_workbook(temp_xls_file.name) + childrens_survey_sheet = workbook["childrens_survey_en"] + labels = {r[0].value: r[1].value for r in childrens_survey_sheet.columns} + self.assertEqual(labels["name"], "1. What is your name?") + self.assertEqual(labels["age"], "2. How old are you?") + + children_sheet = workbook["children"] + labels = {r[0].value: r[1].value for r in children_sheet.columns} + self.assertEqual(labels["fav_colors/red"], "fav_colors/Red") + self.assertEqual(labels["fav_colors/blue"], "fav_colors/Blue") + + # pylint: disable=invalid-name def test_xls_export_with_swahili_labels(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_sw.xlsx'), - default_name='childrens_survey_sw') - # default_language is set to swahili - self.assertEqual( - survey.to_json_dict().get('default_language'), 'swahili' + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_sw.xlsx"), + default_name="childrens_survey_sw", ) + # default_language is set to swahili + self.assertEqual(survey.to_json_dict().get("default_language"), "swahili") export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - export_builder.to_xls_export(temp_xls_file.name, self.data) - temp_xls_file.seek(0) - wb = load_workbook(temp_xls_file.name) - childrens_survey_sheet = wb["childrens_survey_sw"] - labels = dict([(r[0].value, r[1].value) - for r in childrens_survey_sheet.columns]) - self.assertEqual(labels['name'], '1. Jina lako ni?') - self.assertEqual(labels['age'], '2. Umri wako ni?') - - children_sheet = wb["children"] - labels = dict([(r[0].value, r[1].value) - for r in children_sheet.columns]) - self.assertEqual(labels['fav_colors/red'], 'fav_colors/Nyekundu') - self.assertEqual(labels['fav_colors/blue'], 'fav_colors/Bluu') - temp_xls_file.close() - + with NamedTemporaryFile(suffix=".xlsx") as temp_xls_file: + export_builder.to_xls_export(temp_xls_file.name, self.data) + temp_xls_file.seek(0) + workbook = load_workbook(temp_xls_file.name) + childrens_survey_sheet = workbook["childrens_survey_sw"] + labels = {r[0].value: r[1].value for r in childrens_survey_sheet.columns} + self.assertEqual(labels["name"], "1. Jina lako ni?") + self.assertEqual(labels["age"], "2. Umri wako ni?") + + children_sheet = workbook["children"] + labels = {r[0].value: r[1].value for r in children_sheet.columns} + self.assertEqual(labels["fav_colors/red"], "fav_colors/Nyekundu") + self.assertEqual(labels["fav_colors/blue"], "fav_colors/Bluu") + + # pylint: disable=invalid-name def test_csv_export_with_swahili_labels(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_sw.xlsx'), - default_name='childrens_survey_sw') - # default_language is set to swahili - self.assertEqual( - survey.to_json_dict().get('default_language'), 'swahili' + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_sw.xlsx"), + default_name="childrens_survey_sw", ) + # default_language is set to swahili + self.assertEqual(survey.to_json_dict().get("default_language"), "swahili") dd = DataDictionary() + # pylint: disable=protected-access dd._survey = survey ordered_columns = OrderedDict() + # pylint: disable=protected-access CSVDataFrameBuilder._build_ordered_columns(survey, ordered_columns) - ordered_columns['children/fav_colors/red'] = None - labels = get_labels_from_columns(ordered_columns, dd, '/') - self.assertIn('1. Jina lako ni?', labels) - self.assertIn('2. Umri wako ni?', labels) - self.assertIn('fav_colors/Nyekundu', labels) + ordered_columns["children/fav_colors/red"] = None + labels = get_labels_from_columns(ordered_columns, dd, "/") + self.assertIn("1. Jina lako ni?", labels) + self.assertIn("2. Umri wako ni?", labels) + self.assertIn("fav_colors/Nyekundu", labels) # use language provided in keyword argument - labels = get_labels_from_columns(ordered_columns, dd, '/', - language='english') - self.assertIn('1. What is your name?', labels) - self.assertIn('2. How old are you?', labels) - self.assertIn('fav_colors/Red', labels) + labels = get_labels_from_columns(ordered_columns, dd, "/", language="english") + self.assertIn("1. What is your name?", labels) + self.assertIn("2. How old are you?", labels) + self.assertIn("fav_colors/Red", labels) # use default language when language supplied does not exist - labels = get_labels_from_columns(ordered_columns, dd, '/', - language="Chinese") - self.assertIn('1. Jina lako ni?', labels) - self.assertIn('2. Umri wako ni?', labels) - self.assertIn('fav_colors/Nyekundu', labels) + labels = get_labels_from_columns(ordered_columns, dd, "/", language="Chinese") + self.assertIn("1. Jina lako ni?", labels) + self.assertIn("2. Umri wako ni?", labels) + self.assertIn("fav_colors/Nyekundu", labels) def test_select_multiples_choices(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_sw.xlsx'), - default_name='childrens_survey_sw') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_sw.xlsx"), + default_name="childrens_survey_sw", + ) dd = DataDictionary() + # pylint: disable=protected-access dd._survey = survey export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - child = [e for e in dd.get_survey_elements_with_choices() - if e.bind.get('type') == SELECT_BIND_TYPE - and e.type == MULTIPLE_SELECT_TYPE][0] + child = [ + e + for e in dd.get_survey_elements_with_choices() + if e.bind.get("type") == SELECT_BIND_TYPE and e.type == MULTIPLE_SELECT_TYPE + ][0] self.assertNotEqual(child.children, []) + # pylint: disable=protected-access choices = export_builder._get_select_mulitples_choices( - child, dd, ExportBuilder.GROUP_DELIMITER, - ExportBuilder.TRUNCATE_GROUP_TITLE + child, dd, ExportBuilder.GROUP_DELIMITER, ExportBuilder.TRUNCATE_GROUP_TITLE ) expected_choices = [ { - '_label': 'Nyekundu', - '_label_xpath': 'fav_colors/Nyekundu', - 'xpath': 'children/fav_colors/red', - 'title': 'children/fav_colors/red', - 'type': 'string', - 'label': 'fav_colors/Nyekundu' - }, { - '_label': 'Bluu', - '_label_xpath': 'fav_colors/Bluu', - 'xpath': 'children/fav_colors/blue', - 'title': 'children/fav_colors/blue', - 'type': 'string', 'label': 'fav_colors/Bluu' - }, { - '_label': 'Pink', - '_label_xpath': 'fav_colors/Pink', - 'xpath': 'children/fav_colors/pink', - 'title': 'children/fav_colors/pink', - 'type': 'string', 'label': 'fav_colors/Pink' - } + "_label": "Nyekundu", + "_label_xpath": "fav_colors/Nyekundu", + "xpath": "children/fav_colors/red", + "title": "children/fav_colors/red", + "type": "string", + "label": "fav_colors/Nyekundu", + }, + { + "_label": "Bluu", + "_label_xpath": "fav_colors/Bluu", + "xpath": "children/fav_colors/blue", + "title": "children/fav_colors/blue", + "type": "string", + "label": "fav_colors/Bluu", + }, + { + "_label": "Pink", + "_label_xpath": "fav_colors/Pink", + "xpath": "children/fav_colors/pink", + "title": "children/fav_colors/pink", + "type": "string", + "label": "fav_colors/Pink", + }, ] self.assertEqual(choices, expected_choices) select_multiples = { - 'children/fav_colors': [ - ('children/fav_colors/red', 'red', 'Nyekundu'), - ('children/fav_colors/blue', 'blue', 'Bluu'), - ('children/fav_colors/pink', 'pink', 'Pink') - ], 'children/ice.creams': [ - ('children/ice.creams/vanilla', 'vanilla', 'Vanilla'), - ('children/ice.creams/strawberry', 'strawberry', 'Strawberry'), - ('children/ice.creams/chocolate', 'chocolate', 'Chocolate'), - ] + "children/fav_colors": [ + ("children/fav_colors/red", "red", "Nyekundu"), + ("children/fav_colors/blue", "blue", "Bluu"), + ("children/fav_colors/pink", "pink", "Pink"), + ], + "children/ice.creams": [ + ("children/ice.creams/vanilla", "vanilla", "Vanilla"), + ("children/ice.creams/strawberry", "strawberry", "Strawberry"), + ("children/ice.creams/chocolate", "chocolate", "Chocolate"), + ], } - self.assertEqual(CSVDataFrameBuilder._collect_select_multiples(dd), - select_multiples) + self.assertEqual( + CSVDataFrameBuilder._collect_select_multiples(dd), select_multiples + ) + # pylint: disable=invalid-name def test_select_multiples_choices_with_choice_filter(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'choice_filter.xlsx' - ), default_name='choice_filter') + survey = create_survey_from_xls( + _logger_fixture_path("choice_filter.xlsx"), default_name="choice_filter" + ) dd = DataDictionary() + # pylint: disable=protected-access dd._survey = survey export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - child = [e for e in dd.get_survey_elements_with_choices() - if e.bind.get('type') == SELECT_BIND_TYPE - and e.type == MULTIPLE_SELECT_TYPE][0] + child = [ + e + for e in dd.get_survey_elements_with_choices() + if e.bind.get("type") == SELECT_BIND_TYPE and e.type == MULTIPLE_SELECT_TYPE + ][0] + # pylint: disable=protected-access choices = export_builder._get_select_mulitples_choices( - child, dd, ExportBuilder.GROUP_DELIMITER, - ExportBuilder.TRUNCATE_GROUP_TITLE + child, dd, ExportBuilder.GROUP_DELIMITER, ExportBuilder.TRUNCATE_GROUP_TITLE ) expected_choices = [ { - '_label': 'King', - '_label_xpath': 'county/King', - 'label': 'county/King', - 'title': 'county/king', - 'type': 'string', - 'xpath': 'county/king' + "_label": "King", + "_label_xpath": "county/King", + "label": "county/King", + "title": "county/king", + "type": "string", + "xpath": "county/king", }, { - '_label': 'Pierce', - '_label_xpath': 'county/Pierce', - 'label': 'county/Pierce', - 'title': 'county/pierce', - 'type': 'string', - 'xpath': 'county/pierce' + "_label": "Pierce", + "_label_xpath": "county/Pierce", + "label": "county/Pierce", + "title": "county/pierce", + "type": "string", + "xpath": "county/pierce", }, { - '_label': 'King', - '_label_xpath': 'county/King', - 'label': 'county/King', - 'title': 'county/king', - 'type': 'string', - 'xpath': 'county/king' + "_label": "King", + "_label_xpath": "county/King", + "label": "county/King", + "title": "county/king", + "type": "string", + "xpath": "county/king", }, { - '_label': 'Cameron', - '_label_xpath': 'county/Cameron', - 'label': 'county/Cameron', - 'title': 'county/cameron', - 'type': 'string', - 'xpath': 'county/cameron' - } + "_label": "Cameron", + "_label_xpath": "county/Cameron", + "label": "county/Cameron", + "title": "county/cameron", + "type": "string", + "xpath": "county/cameron", + }, ] self.assertEqual(choices, expected_choices) select_multiples = { - 'county': [ - ('county/king', 'king', 'King'), - ('county/pierce', 'pierce', 'Pierce'), - ('county/king', 'king', 'King'), - ('county/cameron', 'cameron', 'Cameron') + "county": [ + ("county/king", "king", "King"), + ("county/pierce", "pierce", "Pierce"), + ("county/king", "king", "King"), + ("county/cameron", "cameron", "Cameron"), ] } - self.assertEqual(CSVDataFrameBuilder._collect_select_multiples(dd), - select_multiples) + # pylint: disable=protected-access + self.assertEqual( + CSVDataFrameBuilder._collect_select_multiples(dd), select_multiples + ) + # pylint: disable=invalid-name def test_string_to_date_with_xls_validation(self): # test "2016-11-02" val = string_to_date_with_xls_validation("2016-11-02") @@ -2505,123 +2679,147 @@ def _create_osm_survey(self): """ # publish form - osm_fixtures_dir = os.path.join(settings.PROJECT_ROOT, 'apps', 'api', - 'tests', 'fixtures', 'osm') - xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') + osm_fixtures_dir = os.path.join( + settings.PROJECT_ROOT, "apps", "api", "tests", "fixtures", "osm" + ) + xlsform_path = os.path.join(osm_fixtures_dir, "osm.xlsx") self._publish_xls_file_and_set_xform(xlsform_path) # make submissions filenames = [ - 'OSMWay234134797.osm', - 'OSMWay34298972.osm', + "OSMWay234134797.osm", + "OSMWay34298972.osm", ] - paths = [os.path.join(osm_fixtures_dir, filename) - for filename in filenames] - submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') + paths = [os.path.join(osm_fixtures_dir, filename) for filename in filenames] + submission_path = os.path.join(osm_fixtures_dir, "instance_a.xml") self._make_submission_w_attachment(submission_path, paths) survey = create_survey_from_xls( - xlsform_path, - default_name=xlsform_path.split('/')[-1].split('.')[0]) + xlsform_path, default_name=xlsform_path.split("/")[-1].split(".")[0] + ) return survey + # pylint: disable=invalid-name def test_zip_csv_export_has_submission_review_fields(self): """ Test that review comment, status and date fields are in csv exports """ - self._create_user_and_login('dave', '1234') + self._create_user_and_login("dave", "1234") survey = self._create_osm_survey() xform = self.xform export_builder = ExportBuilder() export_builder.INCLUDE_REVIEW = True export_builder.set_survey(survey, xform, include_reviews=True) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - export_builder.to_zipped_csv(temp_zip_file.name, self.data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_csv(temp_zip_file.name, self.data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) + # check file's contents - with open(os.path.join(temp_dir, "osm.csv")) as csv_file: - reader = csv.reader(csv_file) - rows = [row for row in reader] + with open(os.path.join(temp_dir, "osm.csv"), encoding="utf-8") as csv_file: + rows = list(csv.reader(csv_file)) actual_headers = rows[0] self.assertIn(REVIEW_COMMENT, sorted(actual_headers)) self.assertIn(REVIEW_DATE, sorted(actual_headers)) self.assertIn(REVIEW_STATUS, sorted(actual_headers)) submission = rows[1] - self.assertEqual(submission[29], 'Rejected') - self.assertEqual(submission[30], 'Wrong Location') - self.assertEqual(submission[31], '2021-05-25T02:27:19') + self.assertEqual(submission[29], "Rejected") + self.assertEqual(submission[30], "Wrong Location") + self.assertEqual(submission[31], "2021-05-25T02:27:19") # check that red and blue are set to true shutil.rmtree(temp_dir) + # pylint: disable=invalid-name def test_xls_export_has_submission_review_fields(self): """ Test that review comment, status and date fields are in xls exports """ - self._create_user_and_login('dave', '1234') + self._create_user_and_login("dave", "1234") survey = self._create_osm_survey() xform = self.xform export_builder = ExportBuilder() export_builder.INCLUDE_REVIEW = True export_builder.set_survey(survey, xform, include_reviews=True) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - export_builder.to_xls_export(temp_xls_file.name, self.osm_data) - temp_xls_file.seek(0) - wb = load_workbook(temp_xls_file.name) - osm_data_sheet = wb["osm"] - rows = [row for row in osm_data_sheet.rows] - xls_headers = [a.value for a in rows[0]] - xls_data = [a.value for a in rows[1]] - temp_xls_file.close() + with NamedTemporaryFile(suffix=".xlsx") as temp_xls_file: + export_builder.to_xls_export(temp_xls_file.name, self.osm_data) + temp_xls_file.seek(0) + workbook = load_workbook(temp_xls_file.name) + osm_data_sheet = workbook["osm"] + rows = list(osm_data_sheet.rows) + xls_headers = [a.value for a in rows[0]] + xls_data = [a.value for a in rows[1]] self.assertIn(REVIEW_COMMENT, sorted(xls_headers)) self.assertIn(REVIEW_DATE, sorted(xls_headers)) self.assertIn(REVIEW_STATUS, sorted(xls_headers)) - self.assertEqual(xls_data[29], 'Rejected') - self.assertEqual(xls_data[30], 'Wrong Location') - self.assertEqual(xls_data[31], '2021-05-25T02:27:19') + self.assertEqual(xls_data[29], "Rejected") + self.assertEqual(xls_data[30], "Wrong Location") + self.assertEqual(xls_data[31], "2021-05-25T02:27:19") + # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_has_submission_review_fields(self): """ Test that review comment, status and date fields are in csv exports """ - self._create_user_and_login('dave', '1234') + self._create_user_and_login("dave", "1234") survey = self._create_osm_survey() xform = self.xform export_builder = ExportBuilder() export_builder.INCLUDE_REVIEW = True export_builder.set_survey(survey, xform, include_reviews=True) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - export_builder.to_zipped_sav(temp_zip_file.name, self.osm_data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() - - with SavReader(os.path.join(temp_dir, "osm.sav"), - returnHeader=True) as reader: - rows = [r for r in reader] - expected_column_headers = [x.encode('utf-8') for x in [ - 'photo', 'osm_road', 'osm_building', 'fav_color', - 'form_completed', 'meta.instanceID', '@_id', '@_uuid', - '@_submission_time', '@_index', '@_parent_table_name', - '@_review_comment', f'@{REVIEW_DATE}', '@_review_status', - '@_parent_index', '@_tags', '@_notes', '@_version', - '@_duration', '@_submitted_by', 'osm_road_ctr_lat', - 'osm_road_ctr_lon', 'osm_road_highway', 'osm_road_lanes', - 'osm_road_name', 'osm_road_way_id', 'osm_building_building', - 'osm_building_building_levels', 'osm_building_ctr_lat', - 'osm_building_ctr_lon', 'osm_building_name', - 'osm_building_way_id']] + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, self.osm_data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) + + with SavReader(os.path.join(temp_dir, "osm.sav"), returnHeader=True) as reader: + rows = list(reader) + expected_column_headers = [ + x.encode("utf-8") + for x in [ + "photo", + "osm_road", + "osm_building", + "fav_color", + "form_completed", + "meta.instanceID", + "@_id", + "@_uuid", + "@_submission_time", + "@_index", + "@_parent_table_name", + "@_review_comment", + f"@{REVIEW_DATE}", + "@_review_status", + "@_parent_index", + "@_tags", + "@_notes", + "@_version", + "@_duration", + "@_submitted_by", + "osm_road_ctr_lat", + "osm_road_ctr_lon", + "osm_road_highway", + "osm_road_lanes", + "osm_road_name", + "osm_road_way_id", + "osm_building_building", + "osm_building_building_levels", + "osm_building_ctr_lat", + "osm_building_ctr_lon", + "osm_building_name", + "osm_building_way_id", + ] + ] self.assertEqual(sorted(rows[0]), sorted(expected_column_headers)) - self.assertEqual(rows[1][29], b'Rejected') - self.assertEqual(rows[1][30], b'Wrong Location') - self.assertEqual(rows[1][31], b'2021-05-25T02:27:19') + self.assertEqual(rows[1][29], b"Rejected") + self.assertEqual(rows[1][30], b"Wrong Location") + self.assertEqual(rows[1][31], b"2021-05-25T02:27:19") + # pylint: disable=invalid-name def test_zipped_csv_export_with_osm_data(self): """ Tests that osm data is included in zipped csv export @@ -2630,93 +2828,134 @@ def test_zipped_csv_export_with_osm_data(self): xform = self.xform export_builder = ExportBuilder() export_builder.set_survey(survey, xform) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - export_builder.to_zipped_csv(temp_zip_file.name, self.osm_data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() - - with open(os.path.join(temp_dir, "osm.csv")) as csv_file: - reader = csv.reader(csv_file) - rows = [row for row in reader] + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_csv(temp_zip_file.name, self.osm_data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) + + with open(os.path.join(temp_dir, "osm.csv"), encoding="utf-8") as csv_file: + rows = list(csv.reader(csv_file)) expected_column_headers = [ - 'photo', 'osm_road', 'osm_building', 'fav_color', - 'form_completed', 'meta/instanceID', '_id', '_uuid', - '_submission_time', '_index', '_parent_table_name', - '_parent_index', '_tags', '_notes', '_version', '_duration', - '_submitted_by', 'osm_road:ctr:lat', 'osm_road:ctr:lon', - 'osm_road:highway', 'osm_road:lanes', 'osm_road:name', - 'osm_road:way:id', 'osm_building:building', - 'osm_building:building:levels', 'osm_building:ctr:lat', - 'osm_building:ctr:lon', 'osm_building:name', - 'osm_building:way:id'] + "photo", + "osm_road", + "osm_building", + "fav_color", + "form_completed", + "meta/instanceID", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + "osm_road:ctr:lat", + "osm_road:ctr:lon", + "osm_road:highway", + "osm_road:lanes", + "osm_road:name", + "osm_road:way:id", + "osm_building:building", + "osm_building:building:levels", + "osm_building:ctr:lat", + "osm_building:ctr:lon", + "osm_building:name", + "osm_building:way:id", + ] self.assertEqual(sorted(rows[0]), sorted(expected_column_headers)) - self.assertEqual(rows[1][0], '1424308569120.jpg') - self.assertEqual(rows[1][1], 'OSMWay234134797.osm') - self.assertEqual(rows[1][2], '23.708174238006087') - self.assertEqual(rows[1][4], 'tertiary') - self.assertEqual(rows[1][6], 'Patuatuli Road') - self.assertEqual(rows[1][13], 'kol') - + self.assertEqual(rows[1][0], "1424308569120.jpg") + self.assertEqual(rows[1][1], "OSMWay234134797.osm") + self.assertEqual(rows[1][2], "23.708174238006087") + self.assertEqual(rows[1][4], "tertiary") + self.assertEqual(rows[1][6], "Patuatuli Road") + self.assertEqual(rows[1][13], "kol") + + # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_osm_data(self): """ Test that osm data is included in zipped sav export """ survey = self._create_osm_survey() xform = self.xform - osm_data = [{ - 'photo': '1424308569120.jpg', - 'osm_road': 'OSMWay234134797.osm', - 'osm_building': 'OSMWay34298972.osm', - 'fav_color': 'red', - 'osm_road_ctr_lat': '23.708174238006087', - 'osm_road_ctr_lon': '90.40946505581161', - 'osm_road_highway': 'tertiary', - 'osm_road_lanes': '2', - 'osm_road_name': 'Patuatuli Road', - 'osm_building_building': 'yes', - 'osm_building_building_levels': '4', - 'osm_building_ctr_lat': '23.707316084046038', - 'osm_building_ctr_lon': '90.40849938337506', - 'osm_building_name': 'kol' - }] + osm_data = [ + { + "photo": "1424308569120.jpg", + "osm_road": "OSMWay234134797.osm", + "osm_building": "OSMWay34298972.osm", + "fav_color": "red", + "osm_road_ctr_lat": "23.708174238006087", + "osm_road_ctr_lon": "90.40946505581161", + "osm_road_highway": "tertiary", + "osm_road_lanes": "2", + "osm_road_name": "Patuatuli Road", + "osm_building_building": "yes", + "osm_building_building_levels": "4", + "osm_building_ctr_lat": "23.707316084046038", + "osm_building_ctr_lon": "90.40849938337506", + "osm_building_name": "kol", + } + ] export_builder = ExportBuilder() export_builder.set_survey(survey, xform) - temp_zip_file = NamedTemporaryFile(suffix='.zip') - export_builder.to_zipped_sav(temp_zip_file.name, osm_data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() - - with SavReader(os.path.join(temp_dir, "osm.sav"), - returnHeader=True) as reader: - rows = [r for r in reader] - expected_column_headers = [x.encode('utf-8') for x in [ - 'photo', 'osm_road', 'osm_building', 'fav_color', - 'form_completed', 'meta.instanceID', '@_id', '@_uuid', - '@_submission_time', '@_index', '@_parent_table_name', - '@_parent_index', '@_tags', '@_notes', '@_version', - '@_duration', '@_submitted_by', 'osm_road_ctr_lat', - 'osm_road_ctr_lon', 'osm_road_highway', 'osm_road_lanes', - 'osm_road_name', 'osm_road_way_id', 'osm_building_building', - 'osm_building_building_levels', 'osm_building_ctr_lat', - 'osm_building_ctr_lon', 'osm_building_name', - 'osm_building_way_id']] + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, osm_data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) + + with SavReader(os.path.join(temp_dir, "osm.sav"), returnHeader=True) as reader: + rows = list(reader) + expected_column_headers = [ + x.encode("utf-8") + for x in [ + "photo", + "osm_road", + "osm_building", + "fav_color", + "form_completed", + "meta.instanceID", + "@_id", + "@_uuid", + "@_submission_time", + "@_index", + "@_parent_table_name", + "@_parent_index", + "@_tags", + "@_notes", + "@_version", + "@_duration", + "@_submitted_by", + "osm_road_ctr_lat", + "osm_road_ctr_lon", + "osm_road_highway", + "osm_road_lanes", + "osm_road_name", + "osm_road_way_id", + "osm_building_building", + "osm_building_building_levels", + "osm_building_ctr_lat", + "osm_building_ctr_lon", + "osm_building_name", + "osm_building_way_id", + ] + ] self.assertEqual(sorted(rows[0]), sorted(expected_column_headers)) - self.assertEqual(rows[1][0], b'1424308569120.jpg') - self.assertEqual(rows[1][1], b'OSMWay234134797.osm') - self.assertEqual(rows[1][2], b'23.708174238006087') - self.assertEqual(rows[1][4], b'tertiary') - self.assertEqual(rows[1][6], b'Patuatuli Road') - self.assertEqual(rows[1][13], b'kol') + self.assertEqual(rows[1][0], b"1424308569120.jpg") + self.assertEqual(rows[1][1], b"OSMWay234134797.osm") + self.assertEqual(rows[1][2], b"23.708174238006087") + self.assertEqual(rows[1][4], b"tertiary") + self.assertEqual(rows[1][6], b"Patuatuli Road") + self.assertEqual(rows[1][13], b"kol") def test_show_choice_labels(self): """ @@ -2734,28 +2973,23 @@ def test_show_choice_labels(self): | | fruits | 2 | Orange | | | fruits | 3 | Apple | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:3]] - for row in children_sheet.rows] + result = [[col.value for col in row[:3]] for row in children_sheet.rows] temp_xls_file.close() - expected_result = [['name', 'age', 'fruit'], ['Maria', 25, 'Mango']] + expected_result = [["name", "age", "fruit"], ["Maria", 25, "Mango"]] self.assertEqual(expected_result, result) - def test_show_choice_labels_multi_language(self): # pylint: disable=C0103 + def test_show_choice_labels_multi_language(self): # pylint: disable=invalid-name """ Test show_choice_labels=true for select one questions - multi language form. @@ -2772,29 +3006,24 @@ def test_show_choice_labels_multi_language(self): # pylint: disable=C0103 | | fruits | 2 | Orange | Orange | | | fruits | 3 | Apple | Pomme | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True - export_builder.language = 'French' + export_builder.language = "French" export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:3]] - for row in children_sheet.rows] + result = [[col.value for col in row[:3]] for row in children_sheet.rows] temp_xls_file.close() - expected_result = [['name', 'age', 'fruit'], ['Maria', 25, 'Mangue']] + expected_result = [["name", "age", "fruit"], ["Maria", 25, "Mangue"]] self.assertEqual(expected_result, result) - def test_show_choice_labels_select_multiple(self): # pylint: disable=C0103 + def test_show_choice_labels_select_multiple(self): # pylint: disable=invalid-name """ Test show_choice_labels=true for select multiple questions - split_select_multiples=true. @@ -2811,30 +3040,24 @@ def test_show_choice_labels_select_multiple(self): # pylint: disable=C0103 | | fruits | 2 | Orange | | | fruits | 3 | Apple | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 2' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 2"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:3]] - for row in children_sheet.rows] + result = [[col.value for col in row[:3]] for row in children_sheet.rows] temp_xls_file.close() - expected_result = [['name', 'age', 'fruit'], - ['Maria', 25, 'Mango Orange']] + expected_result = [["name", "age", "fruit"], ["Maria", 25, "Mango Orange"]] self.assertEqual(expected_result, result) - # pylint: disable=C0103 + # pylint: disable=invalid-name def test_show_choice_labels_select_multiple_1(self): """ Test show_choice_labels=true for select multiple questions @@ -2852,30 +3075,24 @@ def test_show_choice_labels_select_multiple_1(self): | | fruits | 2 | Orange | | | fruits | 3 | Apple | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = False export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 2' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 2"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:3]] - for row in children_sheet.rows] + result = [[col.value for col in row[:3]] for row in children_sheet.rows] temp_xls_file.close() - expected_result = [['name', 'age', 'fruit'], - ['Maria', 25, 'Mango Orange']] + expected_result = [["name", "age", "fruit"], ["Maria", 25, "Mango Orange"]] self.assertEqual(expected_result, result) - # pylint: disable=C0103 + # pylint: disable=invalid-name def test_show_choice_labels_select_multiple_2(self): """ Test show_choice_labels=true for select multiple questions @@ -2893,34 +3110,29 @@ def test_show_choice_labels_select_multiple_2(self): | | fruits | 2 | Orange | | | fruits | 3 | Apple | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = True export_builder.VALUE_SELECT_MULTIPLES = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 2' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 2"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:6]] - for row in children_sheet.rows] + result = [[col.value for col in row[:6]] for row in children_sheet.rows] temp_xls_file.close() expected_result = [ - ['name', 'age', 'fruit', 'fruit/Mango', 'fruit/Orange', - 'fruit/Apple'], - ['Maria', 25, 'Mango Orange', 'Mango', 'Orange', None]] + ["name", "age", "fruit", "fruit/Mango", "fruit/Orange", "fruit/Apple"], + ["Maria", 25, "Mango Orange", "Mango", "Orange", None], + ] self.maxDiff = None self.assertEqual(expected_result, result) - # pylint: disable=C0103 + # pylint: disable=invalid-name def test_show_choice_labels_select_multiple_language(self): """ Test show_choice_labels=true for select multiple questions - multi @@ -2938,31 +3150,25 @@ def test_show_choice_labels_select_multiple_language(self): | | fruits | 2 | Orange | Orange | | | fruits | 3 | Apple | Pomme | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = False - export_builder.language = 'Fr' + export_builder.language = "Fr" export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 3' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 3"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:3]] - for row in children_sheet.rows] + result = [[col.value for col in row[:3]] for row in children_sheet.rows] temp_xls_file.close() - expected_result = [['name', 'age', 'fruit'], - ['Maria', 25, 'Mangue Pomme']] + expected_result = [["name", "age", "fruit"], ["Maria", 25, "Mangue Pomme"]] self.assertEqual(expected_result, result) - # pylint: disable=C0103 + # pylint: disable=invalid-name def test_show_choice_labels_select_multiple_language_1(self): """ Test show_choice_labels=true, split_select_multiples=true, for select @@ -2980,33 +3186,28 @@ def test_show_choice_labels_select_multiple_language_1(self): | | fruits | 2 | Orange | Orange | | | fruits | 3 | Apple | Pomme | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = True - export_builder.language = 'Fr' + export_builder.language = "Fr" export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 3' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 3"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:6]] - for row in children_sheet.rows] + result = [[col.value for col in row[:6]] for row in children_sheet.rows] temp_xls_file.close() expected_result = [ - ['name', 'age', 'fruit', 'fruit/Mangue', 'fruit/Orange', - 'fruit/Pomme'], - ['Maria', 25, 'Mangue Pomme', True, False, True]] + ["name", "age", "fruit", "fruit/Mangue", "fruit/Orange", "fruit/Pomme"], + ["Maria", 25, "Mangue Pomme", True, False, True], + ] self.assertEqual(expected_result, result) - # pylint: disable=C0103 + # pylint: disable=invalid-name def test_show_choice_labels_select_multiple_language_2(self): """ Test show_choice_labels=true, split_select_multiples=true, @@ -3025,34 +3226,29 @@ def test_show_choice_labels_select_multiple_language_2(self): | | fruits | 2 | Orange | Orange | | | fruits | 3 | Apple | Pomme | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = True export_builder.BINARY_SELECT_MULTIPLES = True - export_builder.language = 'Fr' + export_builder.language = "Fr" export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 3' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 3"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:6]] - for row in children_sheet.rows] + result = [[col.value for col in row[:6]] for row in children_sheet.rows] temp_xls_file.close() expected_result = [ - ['name', 'age', 'fruit', 'fruit/Mangue', 'fruit/Orange', - 'fruit/Pomme'], - ['Maria', 25, 'Mangue Pomme', 1, 0, 1]] + ["name", "age", "fruit", "fruit/Mangue", "fruit/Orange", "fruit/Pomme"], + ["Maria", 25, "Mangue Pomme", 1, 0, 1], + ] self.assertEqual(expected_result, result) - # pylint: disable=C0103 + # pylint: disable=invalid-name def test_show_choice_labels_select_multiple_language_3(self): """ Test show_choice_labels=true, split_select_multiples=true, @@ -3071,30 +3267,25 @@ def test_show_choice_labels_select_multiple_language_3(self): | | fruits | 2 | Orange | Orange | | | fruits | 3 | Apple | Pomme | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = True export_builder.VALUE_SELECT_MULTIPLES = True - export_builder.language = 'Fr' + export_builder.language = "Fr" export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 3' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 3"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:6]] - for row in children_sheet.rows] + result = [[col.value for col in row[:6]] for row in children_sheet.rows] temp_xls_file.close() expected_result = [ - ['name', 'age', 'fruit', 'fruit/Mangue', 'fruit/Orange', - 'fruit/Pomme'], - ['Maria', 25, 'Mangue Pomme', 'Mangue', None, 'Pomme']] + ["name", "age", "fruit", "fruit/Mangue", "fruit/Orange", "fruit/Pomme"], + ["Maria", 25, "Mangue Pomme", "Mangue", None, "Pomme"], + ] self.assertEqual(expected_result, result) @@ -3127,27 +3318,24 @@ def test_mulsel_export_with_label_choices(self): | | primary | 9 | Other bank | 9 | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = False export_builder.VALUE_SELECT_MULTIPLES = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") - data = [{ - 'banks_deal': '1 2 3 4', - 'primary_bank': '3' - }] # yapf: disable + data = [{"banks_deal": "1 2 3 4", "primary_bank": "3"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:2]] - for row in children_sheet.rows] + result = [[col.value for col in row[:2]] for row in children_sheet.rows] expected_result = [ - [u'banks_deal', u'primary_bank'], - [u'KCB Equity Co-operative CBA', u'Co-operative']] + ["banks_deal", "primary_bank"], + ["KCB Equity Co-operative CBA", "Co-operative"], + ] temp_xls_file.close() self.assertEqual(result, expected_result) diff --git a/onadata/libs/tests/utils/test_export_tools.py b/onadata/libs/tests/utils/test_export_tools.py index 3230c2274d..6db165f9b9 100644 --- a/onadata/libs/tests/utils/test_export_tools.py +++ b/onadata/libs/tests/utils/test_export_tools.py @@ -8,6 +8,7 @@ import zipfile from builtins import open from datetime import date, datetime, timedelta +from unittest import skipIf from django.conf import settings from django.contrib.sites.models import Site @@ -18,7 +19,11 @@ from pyxform.builder import create_survey_from_xls from rest_framework import exceptions from rest_framework.authtoken.models import Token -from savReaderWriter import SavWriter + +try: + from savReaderWriter import SavWriter +except ImportError: + SavWriter = None from onadata.apps.api import tests as api_tests from onadata.apps.logger.models import Attachment, Instance, XForm @@ -26,86 +31,87 @@ from onadata.apps.viewer.models.export import Export from onadata.apps.viewer.models.parsed_instance import query_data from onadata.apps.api.viewsets.data_viewset import DataViewSet -from onadata.libs.serializers.merged_xform_serializer import \ - MergedXFormSerializer -from onadata.libs.utils.export_builder import (encode_if_str, - get_value_or_attachment_uri) +from onadata.libs.serializers.merged_xform_serializer import MergedXFormSerializer +from onadata.libs.utils.export_builder import encode_if_str, get_value_or_attachment_uri from onadata.libs.utils.export_tools import ( - ExportBuilder, check_pending_export, generate_attachments_zip_export, - generate_export, generate_kml_export, generate_osm_export, - get_repeat_index_tags, kml_export_data, parse_request_export_options, - should_create_new_export, str_to_bool) + ExportBuilder, + check_pending_export, + generate_attachments_zip_export, + generate_export, + generate_kml_export, + generate_osm_export, + get_repeat_index_tags, + kml_export_data, + parse_request_export_options, + should_create_new_export, + str_to_bool, +) def _logger_fixture_path(*args): - return os.path.join(settings.PROJECT_ROOT, 'libs', 'tests', 'fixtures', - *args) + return os.path.join(settings.PROJECT_ROOT, "libs", "tests", "fixtures", *args) class TestExportTools(TestBase): """ Test export_tools functions. """ + def _create_old_export(self, xform, export_type, options): Export(xform=xform, export_type=export_type, options=options).save() - self.export = Export.objects.filter( - xform=xform, export_type=export_type) + self.export = Export.objects.filter(xform=xform, export_type=export_type) def test_encode_if_str(self): row = {"date": date(1899, 9, 9)} date_str = encode_if_str(row, "date", True) - self.assertEqual(date_str, '1899-09-09') + self.assertEqual(date_str, "1899-09-09") row = {"date": date(2001, 9, 9)} date_str = encode_if_str(row, "date", True) - self.assertEqual(date_str, '2001-09-09') + self.assertEqual(date_str, "2001-09-09") row = {"datetime": datetime(1899, 9, 9)} date_str = encode_if_str(row, "datetime", True) - self.assertEqual(date_str, '1899-09-09T00:00:00') + self.assertEqual(date_str, "1899-09-09T00:00:00") row = {"datetime": datetime(2001, 9, 9)} date_str = encode_if_str(row, "datetime", True) - self.assertEqual(date_str, '2001-09-09T00:00:00') + self.assertEqual(date_str, "2001-09-09T00:00:00") row = {"integer_value": 1} - integer_value = encode_if_str( - row, "integer_value", sav_writer=True) - self.assertEqual(integer_value, '1') + integer_value = encode_if_str(row, "integer_value", sav_writer=True) + self.assertEqual(integer_value, "1") - integer_value = encode_if_str( - row, "integer_value") + integer_value = encode_if_str(row, "integer_value") self.assertEqual(integer_value, 1) def test_generate_osm_export(self): filenames = [ - 'OSMWay234134797.osm', - 'OSMWay34298972.osm', + "OSMWay234134797.osm", + "OSMWay34298972.osm", ] osm_fixtures_dir = os.path.realpath( - os.path.join( - os.path.dirname(api_tests.__file__), 'fixtures', 'osm')) - paths = [ - os.path.join(osm_fixtures_dir, filename) for filename in filenames - ] - xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') - combined_osm_path = os.path.join(osm_fixtures_dir, 'combined.osm') + os.path.join(os.path.dirname(api_tests.__file__), "fixtures", "osm") + ) + paths = [os.path.join(osm_fixtures_dir, filename) for filename in filenames] + xlsform_path = os.path.join(osm_fixtures_dir, "osm.xlsx") + combined_osm_path = os.path.join(osm_fixtures_dir, "combined.osm") self._publish_xls_file_and_set_xform(xlsform_path) - submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') - count = Attachment.objects.filter(extension='osm').count() + submission_path = os.path.join(osm_fixtures_dir, "instance_a.xml") + count = Attachment.objects.filter(extension="osm").count() self._make_submission_w_attachment(submission_path, paths) - self.assertTrue( - Attachment.objects.filter(extension='osm').count() > count) + self.assertTrue(Attachment.objects.filter(extension="osm").count() > count) options = {"extension": Attachment.OSM} - export = generate_osm_export(Attachment.OSM, self.user.username, - self.xform.id_string, None, options) + export = generate_osm_export( + Attachment.OSM, self.user.username, self.xform.id_string, None, options + ) self.assertTrue(export.is_successful) - with open(combined_osm_path, encoding='utf-8') as f: + with open(combined_osm_path, encoding="utf-8") as f: osm = f.read() with default_storage.open(export.filepath) as f2: - content = f2.read().decode('utf-8') + content = f2.read().decode("utf-8") self.assertMultiLineEqual(content.strip(), osm.strip()) # delete submission and check that content is no longer in export @@ -113,37 +119,35 @@ def test_generate_osm_export(self): submission.deleted_at = timezone.now() submission.save() - export = generate_osm_export(Attachment.OSM, self.user.username, - self.xform.id_string, None, options) + export = generate_osm_export( + Attachment.OSM, self.user.username, self.xform.id_string, None, options + ) self.assertTrue(export.is_successful) with default_storage.open(export.filepath) as f2: content = f2.read() - self.assertEqual(content, b'') + self.assertEqual(content, b"") def test_generate_attachments_zip_export(self): filenames = [ - 'OSMWay234134797.osm', - 'OSMWay34298972.osm', + "OSMWay234134797.osm", + "OSMWay34298972.osm", ] osm_fixtures_dir = os.path.realpath( - os.path.join( - os.path.dirname(api_tests.__file__), 'fixtures', 'osm')) - paths = [ - os.path.join(osm_fixtures_dir, filename) for filename in filenames - ] - xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') + os.path.join(os.path.dirname(api_tests.__file__), "fixtures", "osm") + ) + paths = [os.path.join(osm_fixtures_dir, filename) for filename in filenames] + xlsform_path = os.path.join(osm_fixtures_dir, "osm.xlsx") self._publish_xls_file_and_set_xform(xlsform_path) - submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') - count = Attachment.objects.filter(extension='osm').count() + submission_path = os.path.join(osm_fixtures_dir, "instance_a.xml") + count = Attachment.objects.filter(extension="osm").count() self._make_submission_w_attachment(submission_path, paths) - self.assertTrue( - Attachment.objects.filter(extension='osm').count() > count) + self.assertTrue(Attachment.objects.filter(extension="osm").count() > count) options = {"extension": Export.ZIP_EXPORT} export = generate_attachments_zip_export( - Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, - options) + Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options + ) self.assertTrue(export.is_successful) @@ -153,8 +157,7 @@ def test_generate_attachments_zip_export(self): zip_file.close() for a in Attachment.objects.all(): - self.assertTrue( - os.path.exists(os.path.join(temp_dir, a.media_file.name))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir) # deleted submission @@ -163,8 +166,8 @@ def test_generate_attachments_zip_export(self): submission.save() export = generate_attachments_zip_export( - Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, - options) + Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options + ) self.assertTrue(export.is_successful) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r") @@ -172,8 +175,7 @@ def test_generate_attachments_zip_export(self): zip_file.close() for a in Attachment.objects.all(): - self.assertFalse( - os.path.exists(os.path.join(temp_dir, a.media_file.name))) + self.assertFalse(os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir) def test_should_create_new_export(self): @@ -184,7 +186,8 @@ def test_should_create_new_export(self): self._publish_transportation_form_and_submit_instance() will_create_new_export = should_create_new_export( - self.xform, export_type, options) + self.xform, export_type, options + ) self.assertTrue(will_create_new_export) @@ -192,7 +195,8 @@ def test_should_create_new_export(self): # deleted self.xform.instances.first().delete() will_create_new_export = should_create_new_export( - self.xform, export_type, options) + self.xform, export_type, options + ) self.assertTrue(will_create_new_export) def test_should_create_export_when_submission_deleted(self): @@ -205,44 +209,45 @@ def test_should_create_export_when_submission_deleted(self): options = { "group_delimiter": "/", "remove_group_name": False, - "split_select_multiples": True + "split_select_multiples": True, } - submission_count = self.xform.instances.filter( - deleted_at__isnull=True).count() + submission_count = self.xform.instances.filter(deleted_at__isnull=True).count() self._create_old_export(self.xform, export_type, options) # Delete submission instance = self.xform.instances.first() instance.set_deleted(datetime.now(), self.user) self.assertEqual( - submission_count - 1, self.xform.instances.filter( - deleted_at__isnull=True).count()) + submission_count - 1, + self.xform.instances.filter(deleted_at__isnull=True).count(), + ) will_create_new_export = should_create_new_export( - self.xform, export_type, options) + self.xform, export_type, options + ) self.assertTrue(will_create_new_export) self._create_old_export(self.xform, export_type, options) # Deleting submission via the API still triggers a new export # when requested - instance_id = self.xform.instances.filter( - deleted_at__isnull=True).first().id - view = DataViewSet.as_view( - {'delete': 'destroy'} - ) + instance_id = self.xform.instances.filter(deleted_at__isnull=True).first().id + view = DataViewSet.as_view({"delete": "destroy"}) token = Token.objects.get(user=self.user) - data = {'instance_ids': [instance_id]} + data = {"instance_ids": [instance_id]} request = self.factory.delete( - '/', data=data, HTTP_AUTHORIZATION=f'Token {token}') + "/", data=data, HTTP_AUTHORIZATION=f"Token {token}" + ) response = view(request, pk=self.xform.id) self.assertEqual(response.status_code, 200) self.assertEqual( - submission_count - 2, self.xform.instances.filter( - deleted_at__isnull=True).count()) + submission_count - 2, + self.xform.instances.filter(deleted_at__isnull=True).count(), + ) will_create_new_export = should_create_new_export( - self.xform, export_type, options) + self.xform, export_type, options + ) self.assertTrue(will_create_new_export) @@ -252,12 +257,13 @@ def test_should_not_create_new_export_when_old_exists(self): options = { "group_delimiter": "/", "remove_group_name": False, - "split_select_multiples": True + "split_select_multiples": True, } self._create_old_export(self.xform, export_type, options) will_create_new_export = should_create_new_export( - self.xform, export_type, options) + self.xform, export_type, options + ) self.assertFalse(will_create_new_export) @@ -266,164 +272,174 @@ def test_should_create_new_export_when_filter_defined(self): options = { "group_delimiter": "/", "remove_group_name": False, - "split_select_multiples": True + "split_select_multiples": True, } self._publish_transportation_form_and_submit_instance() self._create_old_export(self.xform, export_type, options) # Call should_create_new_export with updated options - options['remove_group_name'] = True + options["remove_group_name"] = True will_create_new_export = should_create_new_export( - self.xform, export_type, options) + self.xform, export_type, options + ) self.assertTrue(will_create_new_export) def test_get_value_or_attachment_uri(self): path = os.path.join( - os.path.dirname(__file__), 'fixtures', - 'photo_type_in_repeat_group.xlsx') + os.path.dirname(__file__), "fixtures", "photo_type_in_repeat_group.xlsx" + ) self._publish_xls_file_and_set_xform(path) - filename = u'bob/attachments/123.jpg' - download_url = u'/api/v1/files/1?filename=%s' % filename + filename = "bob/attachments/123.jpg" + download_url = "/api/v1/files/1?filename=%s" % filename # used a smaller version of row because we only using _attachmets key row = { - u'_attachments': [{ - u'mimetype': u'image/jpeg', - u'medium_download_url': u'%s&suffix=medium' % download_url, - u'download_url': download_url, - u'filename': filename, - u'name': '123.jpg', - u'instance': 1, - u'small_download_url': u'%s&suffix=small' % download_url, - u'id': 1, - u'xform': 1 - }] + "_attachments": [ + { + "mimetype": "image/jpeg", + "medium_download_url": "%s&suffix=medium" % download_url, + "download_url": download_url, + "filename": filename, + "name": "123.jpg", + "instance": 1, + "small_download_url": "%s&suffix=small" % download_url, + "id": 1, + "xform": 1, + } + ] } # yapf: disable # when include_images is True, you get the attachment url - media_xpaths = ['photo'] + media_xpaths = ["photo"] attachment_list = None - key = 'photo' - value = u'123.jpg' - val_or_url = get_value_or_attachment_uri(key, value, row, self.xform, - media_xpaths, attachment_list) + key = "photo" + value = "123.jpg" + val_or_url = get_value_or_attachment_uri( + key, value, row, self.xform, media_xpaths, attachment_list + ) self.assertTrue(val_or_url) current_site = Site.objects.get_current() - url = 'http://%s%s' % (current_site.domain, download_url) + url = "http://%s%s" % (current_site.domain, download_url) self.assertEqual(url, val_or_url) # when include_images is False, you get the value media_xpaths = [] - val_or_url = get_value_or_attachment_uri(key, value, row, self.xform, - media_xpaths, attachment_list) + val_or_url = get_value_or_attachment_uri( + key, value, row, self.xform, media_xpaths, attachment_list + ) self.assertTrue(val_or_url) self.assertEqual(value, val_or_url) # test that when row is an empty dict, the function still returns a # value - row.pop('_attachments', None) + row.pop("_attachments", None) self.assertEqual(row, {}) - media_xpaths = ['photo'] - val_or_url = get_value_or_attachment_uri(key, value, row, self.xform, - media_xpaths, attachment_list) + media_xpaths = ["photo"] + val_or_url = get_value_or_attachment_uri( + key, value, row, self.xform, media_xpaths, attachment_list + ) self.assertTrue(val_or_url) self.assertEqual(value, val_or_url) def test_get_attachment_uri_for_filename_with_space(self): path = os.path.join( - os.path.dirname(__file__), 'fixtures', - 'photo_type_in_repeat_group.xlsx') + os.path.dirname(__file__), "fixtures", "photo_type_in_repeat_group.xlsx" + ) self._publish_xls_file_and_set_xform(path) - filename = u'bob/attachments/1_2_3.jpg' - download_url = u'/api/v1/files/1?filename=%s' % filename + filename = "bob/attachments/1_2_3.jpg" + download_url = "/api/v1/files/1?filename=%s" % filename # used a smaller version of row because we only using _attachmets key row = { - u'_attachments': [{ - u'mimetype': u'image/jpeg', - u'medium_download_url': u'%s&suffix=medium' % download_url, - u'download_url': download_url, - u'filename': filename, - u'name': '1 2 3.jpg', - u'instance': 1, - u'small_download_url': u'%s&suffix=small' % download_url, - u'id': 1, - u'xform': 1 - }] + "_attachments": [ + { + "mimetype": "image/jpeg", + "medium_download_url": "%s&suffix=medium" % download_url, + "download_url": download_url, + "filename": filename, + "name": "1 2 3.jpg", + "instance": 1, + "small_download_url": "%s&suffix=small" % download_url, + "id": 1, + "xform": 1, + } + ] } # yapf: disable # when include_images is True, you get the attachment url - media_xpaths = ['photo'] + media_xpaths = ["photo"] attachment_list = None - key = 'photo' - value = u'1 2 3.jpg' - val_or_url = get_value_or_attachment_uri(key, value, row, self.xform, - media_xpaths, attachment_list) + key = "photo" + value = "1 2 3.jpg" + val_or_url = get_value_or_attachment_uri( + key, value, row, self.xform, media_xpaths, attachment_list + ) self.assertTrue(val_or_url) current_site = Site.objects.get_current() - url = 'http://%s%s' % (current_site.domain, download_url) + url = "http://%s%s" % (current_site.domain, download_url) self.assertEqual(url, val_or_url) def test_parse_request_export_options(self): request = self.factory.get( - '/export_async', + "/export_async", data={ "binary_select_multiples": "true", "do_not_split_select_multiples": "false", "remove_group_name": "false", "include_labels": "false", "include_labels_only": "false", - "include_images": "false" - }) + "include_images": "false", + }, + ) options = parse_request_export_options(request.GET) - self.assertEqual(options['split_select_multiples'], True) - self.assertEqual(options['binary_select_multiples'], True) - self.assertEqual(options['include_labels'], False) - self.assertEqual(options['include_labels_only'], False) - self.assertEqual(options['remove_group_name'], False) - self.assertEqual(options['include_images'], False) + self.assertEqual(options["split_select_multiples"], True) + self.assertEqual(options["binary_select_multiples"], True) + self.assertEqual(options["include_labels"], False) + self.assertEqual(options["include_labels_only"], False) + self.assertEqual(options["remove_group_name"], False) + self.assertEqual(options["include_images"], False) request = self.factory.get( - '/export_async', + "/export_async", data={ "do_not_split_select_multiples": "true", "remove_group_name": "true", "include_labels": "true", "include_labels_only": "true", - "include_images": "true" - }) + "include_images": "true", + }, + ) options = parse_request_export_options(request.GET) - self.assertEqual(options['split_select_multiples'], False) - self.assertEqual(options['include_labels'], True) - self.assertEqual(options['include_labels_only'], True) - self.assertEqual(options['remove_group_name'], True) - self.assertEqual(options['include_images'], True) + self.assertEqual(options["split_select_multiples"], False) + self.assertEqual(options["include_labels"], True) + self.assertEqual(options["include_labels_only"], True) + self.assertEqual(options["remove_group_name"], True) + self.assertEqual(options["include_images"], True) def test_export_not_found(self): export_type = "csv" options = { "group_delimiter": "/", "remove_group_name": False, - "split_select_multiples": True + "split_select_multiples": True, } self._publish_transportation_form_and_submit_instance() self._create_old_export(self.xform, export_type, options) - export = Export( - xform=self.xform, export_type=export_type, options=options) + export = Export(xform=self.xform, export_type=export_type, options=options) export.save() export_id = export.pk @@ -449,47 +465,45 @@ def test_kml_export_data(self): | | fruits | orange | Orange | | | fruits | mango | Mango | """ - xform1 = self._publish_markdown(kml_md, self.user, id_string='a') - xform2 = self._publish_markdown(kml_md, self.user, id_string='b') + xform1 = self._publish_markdown(kml_md, self.user, id_string="a") + xform2 = self._publish_markdown(kml_md, self.user, id_string="b") xml = '-1.28 36.83orange' Instance(xform=xform1, xml=xml).save() xml = '32.85 13.04mango' Instance(xform=xform2, xml=xml).save() data = { - 'xforms': [ + "xforms": [ "http://testserver/api/v1/forms/%s" % xform1.pk, "http://testserver/api/v1/forms/%s" % xform2.pk, ], - 'name': 'Merged Dataset', - 'project': - "http://testserver/api/v1/projects/%s" % xform1.project.pk, + "name": "Merged Dataset", + "project": "http://testserver/api/v1/projects/%s" % xform1.project.pk, } # yapf: disable - request = self.factory.post('/') + request = self.factory.post("/") request.user = self.user - serializer = MergedXFormSerializer( - data=data, context={'request': request}) + serializer = MergedXFormSerializer(data=data, context={"request": request}) self.assertTrue(serializer.is_valid()) serializer.save() - xform = XForm.objects.filter( - pk__gt=xform2.pk, is_merged_dataset=True).first() - expected_data = [{ - 'name': u'a', - 'image_urls': [], - 'lat': -1.28, - 'table': u'
    GPS-1.28 36.83
    Fruitorange
    ', # noqa pylint: disable=C0301 - 'lng': 36.83, - 'id': xform1.instances.all().first().pk - }, { - 'name': u'b', - 'image_urls': [], - 'lat': 32.85, - 'table': - u'
    GPS32.85 13.04
    Fruitmango
    ', # noqa pylint: disable=C0301 - 'lng': 13.04, - 'id': xform2.instances.all().first().pk - }] # yapf: disable - self.assertEqual( - kml_export_data(xform.id_string, xform.user), expected_data) + xform = XForm.objects.filter(pk__gt=xform2.pk, is_merged_dataset=True).first() + expected_data = [ + { + "name": "a", + "image_urls": [], + "lat": -1.28, + "table": '
    GPS-1.28 36.83
    Fruitorange
    ', # noqa pylint: disable=C0301 + "lng": 36.83, + "id": xform1.instances.all().first().pk, + }, + { + "name": "b", + "image_urls": [], + "lat": 32.85, + "table": '
    GPS32.85 13.04
    Fruitmango
    ', # noqa pylint: disable=C0301 + "lng": 13.04, + "id": xform2.instances.all().first().pk, + }, + ] # yapf: disable + self.assertEqual(kml_export_data(xform.id_string, xform.user), expected_data) def test_kml_exports(self): """ @@ -500,15 +514,14 @@ def test_kml_exports(self): "group_delimiter": "/", "remove_group_name": False, "split_select_multiples": True, - "extension": 'kml' + "extension": "kml", } self._publish_transportation_form_and_submit_instance() username = self.xform.user.username id_string = self.xform.id_string - export = generate_kml_export( - export_type, username, id_string, options=options) + export = generate_kml_export(export_type, username, id_string, options=options) self.assertIsNotNone(export) self.assertTrue(export.is_successful) @@ -517,31 +530,28 @@ def test_kml_exports(self): export.delete() export = generate_kml_export( - export_type, - username, - id_string, - export_id=export_id, - options=options) + export_type, username, id_string, export_id=export_id, options=options + ) self.assertIsNotNone(export) self.assertTrue(export.is_successful) def test_str_to_bool(self): self.assertTrue(str_to_bool(True)) - self.assertTrue(str_to_bool('True')) - self.assertTrue(str_to_bool('TRUE')) - self.assertTrue(str_to_bool('true')) - self.assertTrue(str_to_bool('t')) - self.assertTrue(str_to_bool('1')) + self.assertTrue(str_to_bool("True")) + self.assertTrue(str_to_bool("TRUE")) + self.assertTrue(str_to_bool("true")) + self.assertTrue(str_to_bool("t")) + self.assertTrue(str_to_bool("1")) self.assertTrue(str_to_bool(1)) self.assertFalse(str_to_bool(False)) - self.assertFalse(str_to_bool('False')) - self.assertFalse(str_to_bool('F')) - self.assertFalse(str_to_bool('random')) + self.assertFalse(str_to_bool("False")) + self.assertFalse(str_to_bool("F")) + self.assertFalse(str_to_bool("random")) self.assertFalse(str_to_bool(234)) self.assertFalse(str_to_bool(0)) - self.assertFalse(str_to_bool('0')) + self.assertFalse(str_to_bool("0")) def test_get_sav_value_labels(self): md = """ @@ -560,7 +570,7 @@ def test_get_sav_value_labels(self): export_builder.set_survey(survey) export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - expected_data = {'fruit': {'orange': 'Orange', 'mango': 'Mango'}} + expected_data = {"fruit": {"orange": "Orange", "mango": "Mango"}} self.assertEqual(export_builder._get_sav_value_labels(), expected_data) def test_sav_choice_list_with_missing_values(self): @@ -580,7 +590,7 @@ def test_sav_choice_list_with_missing_values(self): export_builder.set_survey(survey) export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - expected_data = {'fruit': {'orange': 'Orange', 'mango': ''}} + expected_data = {"fruit": {"orange": "Orange", "mango": ""}} self.assertEqual(export_builder._get_sav_value_labels(), expected_data) def test_get_sav_value_labels_multi_language(self): @@ -600,11 +610,11 @@ def test_get_sav_value_labels_multi_language(self): export_builder.set_survey(survey) export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - expected_data = {'fruit': {'orange': 'Orange', 'mango': 'Mango'}} + expected_data = {"fruit": {"orange": "Orange", "mango": "Mango"}} self.assertEqual(export_builder._get_sav_value_labels(), expected_data) - export_builder.dd._default_language = 'Swahili' - expected_data = {'fruit': {'orange': 'Chungwa', 'mango': 'Maembe'}} + export_builder.data_dicionary._default_language = "Swahili" + expected_data = {"fruit": {"orange": "Chungwa", "mango": "Maembe"}} self.assertEqual(export_builder._get_sav_value_labels(), expected_data) def test_get_sav_value_labels_for_choice_filter(self): @@ -624,12 +634,14 @@ def test_get_sav_value_labels_for_choice_filter(self): export_builder.set_survey(survey) export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - expected_data = {'fruit': {'orange': 'Orange', 'mango': 'Mango'}} + expected_data = {"fruit": {"orange": "Orange", "mango": "Mango"}} self.assertEqual(export_builder._get_sav_value_labels(), expected_data) + @skipIf(SavWriter is None, "savReaderWriter not supported now.") def test_sav_duplicate_columns(self): - more_than_64_char = "akjasdlsakjdkjsadlsakjgdlsagdgdgdsajdgkjdsdgsj" \ - "adsasdasgdsahdsahdsadgsdf" + more_than_64_char = ( + "akjasdlsakjdkjsadlsakjgdlsagdgdgdsajdgkjdsdgsj" "adsasdasgdsahdsahdsadgsdf" + ) md = """ | survey | | | type | name | label | choice_filter | @@ -651,8 +663,9 @@ def test_sav_duplicate_columns(self): | | fts | orange | Orange | 1 | | | fts | mango | Mango | 1 | """ - md = md.format(more_than_64_char, more_than_64_char, more_than_64_char, - more_than_64_char) + md = md.format( + more_than_64_char, more_than_64_char, more_than_64_char, more_than_64_char + ) survey = self.md_to_pyxform_survey(md) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True @@ -661,14 +674,14 @@ def test_sav_duplicate_columns(self): export_builder.set_survey(survey) for sec in export_builder.sections: - sav_options = export_builder._get_sav_options(sec['elements']) + sav_options = export_builder._get_sav_options(sec["elements"]) sav_file = NamedTemporaryFile(suffix=".sav") # No exception is raised SavWriter(sav_file.name, **sav_options) + @skipIf(SavWriter is None, "savReaderWriter not supported now.") def test_sav_special_char_columns(self): - survey = create_survey_from_xls( - _logger_fixture_path('grains/grains.xlsx')) + survey = create_survey_from_xls(_logger_fixture_path("grains/grains.xlsx")) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) @@ -676,7 +689,7 @@ def test_sav_special_char_columns(self): export_builder.set_survey(survey) for sec in export_builder.sections: - sav_options = export_builder._get_sav_options(sec['elements']) + sav_options = export_builder._get_sav_options(sec["elements"]) sav_file = NamedTemporaryFile(suffix=".sav") # No exception is raised SavWriter(sav_file.name, **sav_options) @@ -690,7 +703,8 @@ def test_retrieving_pending_export(self): xform=self.xform, export_type=Export.CSV_EXPORT, options={}, - task_id="abcsde") + task_id="abcsde", + ) export.save() @@ -715,43 +729,40 @@ def test_get_repeat_index_tags(self): """ self.assertIsNone(get_repeat_index_tags(None)) - self.assertEqual(get_repeat_index_tags('.'), ('.', '.')) - self.assertEqual(get_repeat_index_tags('{,}'), ('{', '}')) + self.assertEqual(get_repeat_index_tags("."), (".", ".")) + self.assertEqual(get_repeat_index_tags("{,}"), ("{", "}")) with self.assertRaises(exceptions.ParseError): - get_repeat_index_tags('p') + get_repeat_index_tags("p") def test_generate_filtered_attachments_zip_export(self): """Test media zip file export filters attachments""" filenames = [ - 'OSMWay234134797.osm', - 'OSMWay34298972.osm', + "OSMWay234134797.osm", + "OSMWay34298972.osm", ] osm_fixtures_dir = os.path.realpath( - os.path.join( - os.path.dirname(api_tests.__file__), 'fixtures', 'osm')) - paths = [ - os.path.join(osm_fixtures_dir, filename) for filename in filenames - ] - xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') + os.path.join(os.path.dirname(api_tests.__file__), "fixtures", "osm") + ) + paths = [os.path.join(osm_fixtures_dir, filename) for filename in filenames] + xlsform_path = os.path.join(osm_fixtures_dir, "osm.xlsx") self._publish_xls_file_and_set_xform(xlsform_path) - submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') - count = Attachment.objects.filter(extension='osm').count() + submission_path = os.path.join(osm_fixtures_dir, "instance_a.xml") + count = Attachment.objects.filter(extension="osm").count() self._make_submission_w_attachment(submission_path, paths) self._make_submission_w_attachment(submission_path, paths) - self.assertTrue( - Attachment.objects.filter(extension='osm').count() > count) + self.assertTrue(Attachment.objects.filter(extension="osm").count() > count) options = { "extension": Export.ZIP_EXPORT, - "query": u'{"_submission_time": {"$lte": "2019-01-13T00:00:00"}}'} + "query": '{"_submission_time": {"$lte": "2019-01-13T00:00:00"}}', + } filter_query = options.get("query") - instance_ids = query_data( - self.xform, fields='["_id"]', query=filter_query) + instance_ids = query_data(self.xform, fields='["_id"]', query=filter_query) export = generate_attachments_zip_export( - Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, - options) + Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options + ) self.assertTrue(export.is_successful) @@ -761,22 +772,20 @@ def test_generate_filtered_attachments_zip_export(self): zip_file.close() filtered_attachments = Attachment.objects.filter( - instance__xform_id=self.xform.pk).filter( - instance_id__in=[i_id['_id'] for i_id in instance_ids]) + instance__xform_id=self.xform.pk + ).filter(instance_id__in=[i_id["_id"] for i_id in instance_ids]) - self.assertNotEqual( - Attachment.objects.count(), filtered_attachments.count()) + self.assertNotEqual(Attachment.objects.count(), filtered_attachments.count()) for a in filtered_attachments: - self.assertTrue( - os.path.exists(os.path.join(temp_dir, a.media_file.name))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir) # export with no query - options.pop('query') + options.pop("query") export1 = generate_attachments_zip_export( - Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, - options) + Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options + ) self.assertTrue(export1.is_successful) @@ -786,6 +795,5 @@ def test_generate_filtered_attachments_zip_export(self): zip_file.close() for a in Attachment.objects.all(): - self.assertTrue( - os.path.exists(os.path.join(temp_dir, a.media_file.name))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir) diff --git a/onadata/libs/tests/utils/test_logger_tools.py b/onadata/libs/tests/utils/test_logger_tools.py index ab81c7f636..7a8a5710b7 100644 --- a/onadata/libs/tests/utils/test_logger_tools.py +++ b/onadata/libs/tests/utils/test_logger_tools.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Test logger_tools utility functions. +""" import os import re from io import BytesIO @@ -11,39 +15,52 @@ from onadata.apps.logger.import_tools import django_file from onadata.apps.logger.models import Instance from onadata.apps.main.tests.test_base import TestBase -from onadata.libs.utils.common_tags import (MEDIA_ALL_RECEIVED, MEDIA_COUNT, - TOTAL_MEDIA) +from onadata.libs.utils.common_tags import MEDIA_ALL_RECEIVED, MEDIA_COUNT, TOTAL_MEDIA from onadata.libs.utils.logger_tools import ( - create_instance, generate_content_disposition_header, get_first_record, - safe_create_instance) + create_instance, + generate_content_disposition_header, + get_first_record, + safe_create_instance, +) from onadata.apps.logger.xform_instance_parser import AttachmentNameError from django.core.files.uploadedfile import InMemoryUploadedFile class TestLoggerTools(PyxformTestCase, TestBase): + """ + Test logger_tools utility functions. + """ + def test_generate_content_disposition_header(self): file_name = "export" extension = "ext" - date_pattern = "\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}" # noqa + date_pattern = "\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}" # noqa file_name_pattern = "%s.%s" % (file_name, extension) - file_name_with_timestamp_pattern = \ - "%s-%s.%s" % (file_name, date_pattern, extension) - return_value_with_no_name = \ - generate_content_disposition_header(None, extension) - self.assertEquals(return_value_with_no_name, "attachment;") + file_name_with_timestamp_pattern = "%s-%s.%s" % ( + file_name, + date_pattern, + extension, + ) + return_value_with_no_name = generate_content_disposition_header(None, extension) + self.assertEqual(return_value_with_no_name, "attachment;") - return_value_with_name_and_no_show_date = \ - generate_content_disposition_header(file_name, extension) + return_value_with_name_and_no_show_date = generate_content_disposition_header( + file_name, extension + ) self.assertTrue( - re.search(file_name_with_timestamp_pattern, - return_value_with_name_and_no_show_date)) + re.search( + file_name_with_timestamp_pattern, + return_value_with_name_and_no_show_date, + ) + ) - return_value_with_name_and_false_show_date = \ + return_value_with_name_and_false_show_date = ( generate_content_disposition_header(file_name, extension, False) + ) self.assertTrue( - re.search(file_name_pattern, - return_value_with_name_and_false_show_date)) + re.search(file_name_pattern, return_value_with_name_and_false_show_date) + ) def test_attachment_tracking(self): """ @@ -68,53 +85,66 @@ def test_attachment_tracking(self): 1300221157303.jpg 1300375832136.jpg     - """.format(self.xform.id_string) - file_path = "{}/apps/logger/tests/Health_2011_03_13."\ - "xml_2011-03-15_20-30-28/1300221157303"\ - ".jpg".format(settings.PROJECT_ROOT) + """.format( + self.xform.id_string + ) + file_path = ( + "{}/apps/logger/tests/Health_2011_03_13." + "xml_2011-03-15_20-30-28/1300221157303" + ".jpg".format(settings.PROJECT_ROOT) + ) media_file = django_file( - path=file_path, field_name="image1", content_type="image/jpeg") + path=file_path, field_name="image1", content_type="image/jpeg" + ) instance = create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media_file], + ) self.assertFalse(instance.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance.json[TOTAL_MEDIA], 2) - self.assertEquals(instance.json[MEDIA_COUNT], 1) - self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media) - self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count) - self.assertEquals(instance.json[MEDIA_ALL_RECEIVED], - instance.media_all_received) - file2_path = "{}/apps/logger/tests/Water_2011_03_17_2011-03-17_16-29"\ - "-59/1300375832136.jpg".format(settings.PROJECT_ROOT) + self.assertEqual(instance.json[TOTAL_MEDIA], 2) + self.assertEqual(instance.json[MEDIA_COUNT], 1) + self.assertEqual(instance.json[TOTAL_MEDIA], instance.total_media) + self.assertEqual(instance.json[MEDIA_COUNT], instance.media_count) + self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) + file2_path = ( + "{}/apps/logger/tests/Water_2011_03_17_2011-03-17_16-29" + "-59/1300375832136.jpg".format(settings.PROJECT_ROOT) + ) media2_file = django_file( - path=file2_path, field_name="image2", content_type="image/jpeg") + path=file2_path, field_name="image2", content_type="image/jpeg" + ) create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media2_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media2_file], + ) instance2 = Instance.objects.get(pk=instance.pk) self.assertTrue(instance2.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance2.json[TOTAL_MEDIA], 2) - self.assertEquals(instance2.json[MEDIA_COUNT], 2) - self.assertEquals(instance2.json[TOTAL_MEDIA], instance2.total_media) - self.assertEquals(instance2.json[MEDIA_COUNT], instance2.media_count) - self.assertEquals(instance2.json[MEDIA_ALL_RECEIVED], - instance2.media_all_received) + self.assertEqual(instance2.json[TOTAL_MEDIA], 2) + self.assertEqual(instance2.json[MEDIA_COUNT], 2) + self.assertEqual(instance2.json[TOTAL_MEDIA], instance2.total_media) + self.assertEqual(instance2.json[MEDIA_COUNT], instance2.media_count) + self.assertEqual( + instance2.json[MEDIA_ALL_RECEIVED], instance2.media_all_received + ) media2_file = django_file( - path=file2_path, field_name="image2", content_type="image/jpeg") + path=file2_path, field_name="image2", content_type="image/jpeg" + ) create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media2_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media2_file], + ) instance3 = Instance.objects.get(pk=instance.pk) self.assertTrue(instance3.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance3.json[TOTAL_MEDIA], 2) - self.assertEquals(instance3.json[MEDIA_COUNT], 2) - self.assertEquals(instance3.json[TOTAL_MEDIA], instance2.total_media) - self.assertEquals(instance3.json[MEDIA_COUNT], instance2.media_count) - self.assertEquals(instance3.json[MEDIA_ALL_RECEIVED], - instance3.media_all_received) + self.assertEqual(instance3.json[TOTAL_MEDIA], 2) + self.assertEqual(instance3.json[MEDIA_COUNT], 2) + self.assertEqual(instance3.json[TOTAL_MEDIA], instance2.total_media) + self.assertEqual(instance3.json[MEDIA_COUNT], instance2.media_count) + self.assertEqual( + instance3.json[MEDIA_ALL_RECEIVED], instance3.media_all_received + ) def test_attachment_tracking_for_repeats(self): """ @@ -144,39 +174,49 @@ def test_attachment_tracking_for_repeats(self): 1300375832136.jpg     - """.format(self.xform.id_string) - file_path = "{}/apps/logger/tests/Health_2011_03_13."\ - "xml_2011-03-15_20-30-28/1300221157303"\ - ".jpg".format(settings.PROJECT_ROOT) + """.format( + self.xform.id_string + ) + file_path = ( + "{}/apps/logger/tests/Health_2011_03_13." + "xml_2011-03-15_20-30-28/1300221157303" + ".jpg".format(settings.PROJECT_ROOT) + ) media_file = django_file( - path=file_path, field_name="image1", content_type="image/jpeg") + path=file_path, field_name="image1", content_type="image/jpeg" + ) instance = create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media_file], + ) self.assertFalse(instance.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance.json[TOTAL_MEDIA], 2) - self.assertEquals(instance.json[MEDIA_COUNT], 1) - self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media) - self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count) - self.assertEquals(instance.json[MEDIA_ALL_RECEIVED], - instance.media_all_received) - file2_path = "{}/apps/logger/tests/Water_2011_03_17_2011-03-17_16-29"\ - "-59/1300375832136.jpg".format(settings.PROJECT_ROOT) + self.assertEqual(instance.json[TOTAL_MEDIA], 2) + self.assertEqual(instance.json[MEDIA_COUNT], 1) + self.assertEqual(instance.json[TOTAL_MEDIA], instance.total_media) + self.assertEqual(instance.json[MEDIA_COUNT], instance.media_count) + self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) + file2_path = ( + "{}/apps/logger/tests/Water_2011_03_17_2011-03-17_16-29" + "-59/1300375832136.jpg".format(settings.PROJECT_ROOT) + ) media2_file = django_file( - path=file2_path, field_name="image1", content_type="image/jpeg") + path=file2_path, field_name="image1", content_type="image/jpeg" + ) create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media2_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media2_file], + ) instance2 = Instance.objects.get(pk=instance.pk) self.assertTrue(instance2.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance2.json[TOTAL_MEDIA], 2) - self.assertEquals(instance2.json[MEDIA_COUNT], 2) - self.assertEquals(instance2.json[TOTAL_MEDIA], instance2.total_media) - self.assertEquals(instance2.json[MEDIA_COUNT], instance2.media_count) - self.assertEquals(instance2.json[MEDIA_ALL_RECEIVED], - instance2.media_all_received) + self.assertEqual(instance2.json[TOTAL_MEDIA], 2) + self.assertEqual(instance2.json[MEDIA_COUNT], 2) + self.assertEqual(instance2.json[TOTAL_MEDIA], instance2.total_media) + self.assertEqual(instance2.json[MEDIA_COUNT], instance2.media_count) + self.assertEqual( + instance2.json[MEDIA_ALL_RECEIVED], instance2.media_all_received + ) def test_attachment_tracking_for_nested_repeats(self): """ @@ -208,39 +248,49 @@ def test_attachment_tracking_for_nested_repeats(self): 1300375832136.jpg - """.format(self.xform.id_string) - file_path = "{}/apps/logger/tests/Health_2011_03_13."\ - "xml_2011-03-15_20-30-28/1300221157303"\ - ".jpg".format(settings.PROJECT_ROOT) + """.format( + self.xform.id_string + ) + file_path = ( + "{}/apps/logger/tests/Health_2011_03_13." + "xml_2011-03-15_20-30-28/1300221157303" + ".jpg".format(settings.PROJECT_ROOT) + ) media_file = django_file( - path=file_path, field_name="image1", content_type="image/jpeg") + path=file_path, field_name="image1", content_type="image/jpeg" + ) instance = create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media_file], + ) self.assertFalse(instance.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance.json[TOTAL_MEDIA], 2) - self.assertEquals(instance.json[MEDIA_COUNT], 1) - self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media) - self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count) - self.assertEquals(instance.json[MEDIA_ALL_RECEIVED], - instance.media_all_received) - file2_path = "{}/apps/logger/tests/Water_2011_03_17_2011-03-17_16-29"\ - "-59/1300375832136.jpg".format(settings.PROJECT_ROOT) + self.assertEqual(instance.json[TOTAL_MEDIA], 2) + self.assertEqual(instance.json[MEDIA_COUNT], 1) + self.assertEqual(instance.json[TOTAL_MEDIA], instance.total_media) + self.assertEqual(instance.json[MEDIA_COUNT], instance.media_count) + self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) + file2_path = ( + "{}/apps/logger/tests/Water_2011_03_17_2011-03-17_16-29" + "-59/1300375832136.jpg".format(settings.PROJECT_ROOT) + ) media2_file = django_file( - path=file2_path, field_name="image1", content_type="image/jpeg") + path=file2_path, field_name="image1", content_type="image/jpeg" + ) create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media2_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media2_file], + ) instance2 = Instance.objects.get(pk=instance.pk) self.assertTrue(instance2.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance2.json[TOTAL_MEDIA], 2) - self.assertEquals(instance2.json[MEDIA_COUNT], 2) - self.assertEquals(instance2.json[TOTAL_MEDIA], instance2.total_media) - self.assertEquals(instance2.json[MEDIA_COUNT], instance2.media_count) - self.assertEquals(instance2.json[MEDIA_ALL_RECEIVED], - instance2.media_all_received) + self.assertEqual(instance2.json[TOTAL_MEDIA], 2) + self.assertEqual(instance2.json[MEDIA_COUNT], 2) + self.assertEqual(instance2.json[TOTAL_MEDIA], instance2.total_media) + self.assertEqual(instance2.json[MEDIA_COUNT], instance2.media_count) + self.assertEqual( + instance2.json[MEDIA_ALL_RECEIVED], instance2.media_all_received + ) def test_replaced_attachments_not_tracked(self): """ @@ -266,32 +316,40 @@ def test_replaced_attachments_not_tracked(self): Health_2011_03_13.xml_2011-03-15_20-30-28.xml 1300221157303.jpg - """.format(self.xform.id_string) - media_root = (f'{settings.PROJECT_ROOT}/apps/logger/tests/Health' - '_2011_03_13.xml_2011-03-15_20-30-28/') + """.format( + self.xform.id_string + ) + media_root = ( + f"{settings.PROJECT_ROOT}/apps/logger/tests/Health" + "_2011_03_13.xml_2011-03-15_20-30-28/" + ) image_media = django_file( - path=f'{media_root}1300221157303.jpg', field_name='image', - content_type='image/jpeg') + path=f"{media_root}1300221157303.jpg", + field_name="image", + content_type="image/jpeg", + ) file_media = django_file( - path=f'{media_root}Health_2011_03_13.xml_2011-03-15_20-30-28.xml', - field_name='file', content_type='text/xml') + path=f"{media_root}Health_2011_03_13.xml_2011-03-15_20-30-28.xml", + field_name="file", + content_type="text/xml", + ) instance = create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[file_media, image_media]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[file_media, image_media], + ) self.assertTrue(instance.json[MEDIA_ALL_RECEIVED]) self.assertEqual( - instance.attachments.filter(deleted_at__isnull=True).count(), - 2) - self.assertEquals(instance.json[TOTAL_MEDIA], 2) - self.assertEquals(instance.json[MEDIA_COUNT], 2) - self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media) - self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count) - self.assertEquals(instance.json[MEDIA_ALL_RECEIVED], - instance.media_all_received) - patch_value = 'onadata.apps.logger.models.Instance.get_expected_media' + instance.attachments.filter(deleted_at__isnull=True).count(), 2 + ) + self.assertEqual(instance.json[TOTAL_MEDIA], 2) + self.assertEqual(instance.json[MEDIA_COUNT], 2) + self.assertEqual(instance.json[TOTAL_MEDIA], instance.total_media) + self.assertEqual(instance.json[MEDIA_COUNT], instance.media_count) + self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) + patch_value = "onadata.apps.logger.models.Instance.get_expected_media" with patch(patch_value) as get_expected_media: - get_expected_media.return_value = ['1300375832136.jpg'] + get_expected_media.return_value = ["1300375832136.jpg"] updated_xml_string = """ @@ -301,32 +359,33 @@ def test_replaced_attachments_not_tracked(self): 1300375832136.jpg - """.format(self.xform.id_string) - file2_path = "{}/apps/logger/tests/Water_2011_03_17_2011"\ - "-03-17_16-29-59/1300375832136.jpg".format( - settings.PROJECT_ROOT) + """.format( + self.xform.id_string + ) + file2_path = ( + "{}/apps/logger/tests/Water_2011_03_17_2011" + "-03-17_16-29-59/1300375832136.jpg".format(settings.PROJECT_ROOT) + ) media2_file = django_file( - path=file2_path, - field_name="image1", - content_type="image/jpeg") + path=file2_path, field_name="image1", content_type="image/jpeg" + ) create_instance( self.user.username, - BytesIO(updated_xml_string.strip().encode('utf-8')), - media_files=[media2_file]) + BytesIO(updated_xml_string.strip().encode("utf-8")), + media_files=[media2_file], + ) instance2 = Instance.objects.get(pk=instance.pk) self.assertTrue(instance2.json[MEDIA_ALL_RECEIVED]) # Test that only one attachment is recognised for this submission # Since the file is no longer present in the submission - self.assertEquals(instance2.json[TOTAL_MEDIA], 1) - self.assertEquals(instance2.json[MEDIA_COUNT], 1) - self.assertEquals( - instance2.json[TOTAL_MEDIA], instance2.total_media) - self.assertEquals( - instance2.json[MEDIA_COUNT], instance2.media_count) - self.assertEquals( - instance2.json[MEDIA_ALL_RECEIVED], - instance2.media_all_received) + self.assertEqual(instance2.json[TOTAL_MEDIA], 1) + self.assertEqual(instance2.json[MEDIA_COUNT], 1) + self.assertEqual(instance2.json[TOTAL_MEDIA], instance2.total_media) + self.assertEqual(instance2.json[MEDIA_COUNT], instance2.media_count) + self.assertEqual( + instance2.json[MEDIA_ALL_RECEIVED], instance2.media_all_received + ) def test_attachment_tracking_duplicate(self): """ @@ -350,37 +409,45 @@ def test_attachment_tracking_duplicate(self): 1300221157303.jpg 1300375832136.jpg - """.format(self.xform.id_string) - file_path = "{}/apps/logger/tests/Health_2011_03_13."\ - "xml_2011-03-15_20-30-28/1300221157303"\ - ".jpg".format(settings.PROJECT_ROOT) + """.format( + self.xform.id_string + ) + file_path = ( + "{}/apps/logger/tests/Health_2011_03_13." + "xml_2011-03-15_20-30-28/1300221157303" + ".jpg".format(settings.PROJECT_ROOT) + ) media_file = django_file( - path=file_path, field_name="image1", content_type="image/jpeg") + path=file_path, field_name="image1", content_type="image/jpeg" + ) instance = create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media_file], + ) self.assertFalse(instance.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance.json[TOTAL_MEDIA], 2) - self.assertEquals(instance.json[MEDIA_COUNT], 1) - self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media) - self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count) - self.assertEquals(instance.json[MEDIA_ALL_RECEIVED], - instance.media_all_received) + self.assertEqual(instance.json[TOTAL_MEDIA], 2) + self.assertEqual(instance.json[MEDIA_COUNT], 1) + self.assertEqual(instance.json[TOTAL_MEDIA], instance.total_media) + self.assertEqual(instance.json[MEDIA_COUNT], instance.media_count) + self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) media2_file = django_file( - path=file_path, field_name="image1", content_type="image/jpeg") + path=file_path, field_name="image1", content_type="image/jpeg" + ) create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media2_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media2_file], + ) instance2 = Instance.objects.get(pk=instance.pk) self.assertFalse(instance2.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance2.json[TOTAL_MEDIA], 2) - self.assertEquals(instance2.json[MEDIA_COUNT], 1) - self.assertEquals(instance2.json[TOTAL_MEDIA], instance2.total_media) - self.assertEquals(instance2.json[MEDIA_COUNT], instance2.media_count) - self.assertEquals(instance2.json[MEDIA_ALL_RECEIVED], - instance2.media_all_received) + self.assertEqual(instance2.json[TOTAL_MEDIA], 2) + self.assertEqual(instance2.json[MEDIA_COUNT], 1) + self.assertEqual(instance2.json[TOTAL_MEDIA], instance2.total_media) + self.assertEqual(instance2.json[MEDIA_COUNT], instance2.media_count) + self.assertEqual( + instance2.json[MEDIA_ALL_RECEIVED], instance2.media_all_received + ) def test_attachment_tracking_not_in_submission(self): """ @@ -403,27 +470,35 @@ def test_attachment_tracking_not_in_submission(self): 1300221157303.jpg 1300375832136.jpg - """.format(self.xform.id_string) - file_path = "{}/apps/logger/tests/Health_2011_03_13."\ - "xml_2011-03-15_20-30-28/1300221157303"\ - ".jpg".format(settings.PROJECT_ROOT) - file2_path = "{}/libs/tests/utils/fixtures/tutorial/instances/uuid1/"\ - "1442323232322.jpg".format(settings.PROJECT_ROOT) + """.format( + self.xform.id_string + ) + file_path = ( + "{}/apps/logger/tests/Health_2011_03_13." + "xml_2011-03-15_20-30-28/1300221157303" + ".jpg".format(settings.PROJECT_ROOT) + ) + file2_path = ( + "{}/libs/tests/utils/fixtures/tutorial/instances/uuid1/" + "1442323232322.jpg".format(settings.PROJECT_ROOT) + ) media_file = django_file( - path=file_path, field_name="image1", content_type="image/jpeg") + path=file_path, field_name="image1", content_type="image/jpeg" + ) media2_file = django_file( - path=file2_path, field_name="image1", content_type="image/jpeg") + path=file2_path, field_name="image1", content_type="image/jpeg" + ) instance = create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media_file, media2_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media_file, media2_file], + ) self.assertFalse(instance.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance.json[TOTAL_MEDIA], 2) - self.assertEquals(instance.json[MEDIA_COUNT], 1) - self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media) - self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count) - self.assertEquals(instance.json[MEDIA_ALL_RECEIVED], - instance.media_all_received) + self.assertEqual(instance.json[TOTAL_MEDIA], 2) + self.assertEqual(instance.json[MEDIA_COUNT], 1) + self.assertEqual(instance.json[TOTAL_MEDIA], instance.total_media) + self.assertEqual(instance.json[MEDIA_COUNT], instance.media_count) + self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) def test_get_first_record(self): """ @@ -437,18 +512,21 @@ def test_get_first_record(self): self._create_user_and_login() xform = self._publish_markdown(xform_md, self.user) - self.assertIsNone(get_first_record(Instance.objects.all().only('id'))) + self.assertIsNone(get_first_record(Instance.objects.all().only("id"))) xml_string = """ Alice - """.format(xform.id_string) + """.format( + xform.id_string + ) instance = create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[]) - record = get_first_record(Instance.objects.all().only('id')) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[], + ) + record = get_first_record(Instance.objects.all().only("id")) self.assertIsNotNone(record) self.assertEqual(record.id, instance.id) @@ -457,8 +535,10 @@ def test_check_encryption_status(self): Test that the encryption status of a submission is checked and unencrypted submissions are rejected when made to encrypted forms. """ - form_path = (f"{settings.PROJECT_ROOT}/libs/tests/" - "fixtures/tutorial/tutorial_encrypted.xlsx") + form_path = ( + f"{settings.PROJECT_ROOT}/libs/tests/" + "fixtures/tutorial/tutorial_encrypted.xlsx" + ) self._publish_xls_file_and_set_xform(form_path) instance_xml = f""" 1300221157303.jpg 1300375832136.jpg - """.format(self.xform.id_string) + """.format( + self.xform.id_string + ) - file_path = "{}/apps/logger/tests/Health_2011_03_13."\ - "xml_2011-03-15_20-30-28/1300221157303"\ - ".jpg".format(settings.PROJECT_ROOT) - f = open(file_path, 'rb') + file_path = ( + "{}/apps/logger/tests/Health_2011_03_13." + "xml_2011-03-15_20-30-28/1300221157303" + ".jpg".format(settings.PROJECT_ROOT) + ) + f = open(file_path, "rb") media_file = InMemoryUploadedFile( file=f, field_name="image1", - name=f'{f.name} +\ - test_file_name_test_file_name_test_file_name_test_file_name_test_file_name_test_file_name', # noqa + name=f"{f.name} +\ + test_file_name_test_file_name_test_file_name_test_file_name_test_file_name_test_file_name", # noqa content_type="image/jpeg", size=os.path.getsize(file_path), - charset=None + charset=None, ) with self.assertRaises(AttachmentNameError): create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media_file], + ) diff --git a/onadata/libs/tests/utils/test_middleware.py b/onadata/libs/tests/utils/test_middleware.py index 1e3cb5d62f..b6c7ad8c01 100644 --- a/onadata/libs/tests/utils/test_middleware.py +++ b/onadata/libs/tests/utils/test_middleware.py @@ -1,6 +1,6 @@ from unittest.mock import MagicMock -from django.conf.urls import url +from django.urls import re_path from django.db import OperationalError from django.http import HttpResponse from django.test import RequestFactory, TestCase @@ -13,7 +13,7 @@ def normal_view(request): urlpatterns = [ - url('middleware_exceptions/view/', normal_view, name='normal'), + re_path('middleware_exceptions/view/', normal_view, name='normal'), ] diff --git a/onadata/libs/tests/utils/test_model_tools.py b/onadata/libs/tests/utils/test_model_tools.py index ec187dd476..1848db1d91 100644 --- a/onadata/libs/tests/utils/test_model_tools.py +++ b/onadata/libs/tests/utils/test_model_tools.py @@ -14,7 +14,7 @@ def test_queryset_iterator(self): username='test_2', password='test_2', email='test_2@test.com') user_model.objects.create_user( username='test_3', password='test_3', email='test@test_3.com') - self.assertEquals( + self.assertEqual( 'generator', queryset_iterator( user_model.objects.all(), chunksize=1).__class__.__name__ diff --git a/onadata/libs/tests/utils/test_viewer_tools.py b/onadata/libs/tests/utils/test_viewer_tools.py index 9e0b492bda..2ed76c193f 100644 --- a/onadata/libs/tests/utils/test_viewer_tools.py +++ b/onadata/libs/tests/utils/test_viewer_tools.py @@ -3,19 +3,24 @@ import os from django.core.files.base import File +from django.core.files.temp import NamedTemporaryFile from django.http import Http404 from django.test.client import RequestFactory from django.test.utils import override_settings from django.utils import timezone + from mock import patch -from onadata.apps.logger.models import XForm, Instance, Attachment +from onadata.apps.logger.models import Attachment, Instance, XForm from onadata.apps.main.tests.test_base import TestBase -from onadata.libs.utils.viewer_tools import (create_attachments_zipfile, - export_def_from_filename, - generate_enketo_form_defaults, - get_client_ip, get_form, - get_form_url) +from onadata.libs.utils.viewer_tools import ( + create_attachments_zipfile, + export_def_from_filename, + generate_enketo_form_defaults, + get_client_ip, + get_form, + get_form_url, +) class TestViewerTools(TestBase): @@ -25,8 +30,8 @@ def test_export_def_from_filename(self): """Test export_def_from_filename().""" filename = "path/filename.xlsx" ext, mime_type = export_def_from_filename(filename) - self.assertEqual(ext, 'xlsx') - self.assertEqual(mime_type, 'vnd.openxmlformats') + self.assertEqual(ext, "xlsx") + self.assertEqual(mime_type, "vnd.openxmlformats") def test_get_client_ip(self): """Test get_client_ip().""" @@ -53,14 +58,12 @@ def test_get_enketo_defaults_with_right_xform(self): # create xform self._publish_transportation_form() # create kwargs with existing xform variable - xform_variable_name = \ - 'available_transportation_types_to_referral_facility' - xform_variable_value = 'ambulance' + xform_variable_name = "available_transportation_types_to_referral_facility" + xform_variable_value = "ambulance" kwargs = {xform_variable_name: xform_variable_value} defaults = generate_enketo_form_defaults(self.xform, **kwargs) - key = "defaults[/data/transport/{}]".format( - xform_variable_name) + key = "defaults[/data/transport/{}]".format(xform_variable_name) self.assertEqual(defaults, {key: xform_variable_value}) # pylint: disable=C0103 @@ -69,25 +72,26 @@ def test_get_enketo_defaults_with_multiple_params(self): # create xform self._publish_transportation_form() # create kwargs with existing xform variable - transportation_types = \ - 'available_transportation_types_to_referral_facility' - transportation_types_value = 'ambulance' + transportation_types = "available_transportation_types_to_referral_facility" + transportation_types_value = "ambulance" - frequency = 'frequency_to_referral_facility' - frequency_value = 'daily' + frequency = "frequency_to_referral_facility" + frequency_value = "daily" kwargs = { transportation_types: transportation_types_value, - frequency: frequency_value + frequency: frequency_value, } defaults = generate_enketo_form_defaults(self.xform, **kwargs) - transportation_types_key = \ - "defaults[/data/transport/{}]".format( - transportation_types) - frequency_key = "defaults[/data/transport/"\ - "loop_over_transport_types_frequency/"\ - "{}/{}]".format(transportation_types_value, frequency) + transportation_types_key = "defaults[/data/transport/{}]".format( + transportation_types + ) + frequency_key = ( + "defaults[/data/transport/" + "loop_over_transport_types_frequency/" + "{}/{}]".format(transportation_types_value, frequency) + ) self.assertIn(transportation_types_key, defaults) self.assertIn(frequency_key, defaults) @@ -97,7 +101,7 @@ def test_get_enketo_defaults_with_non_existent_field(self): # create xform self._publish_transportation_form() # create kwargs with NON-existing xform variable - kwargs = {'name': 'bla'} + kwargs = {"name": "bla"} defaults = generate_enketo_form_defaults(self.xform, **kwargs) self.assertEqual(defaults, {}) @@ -105,17 +109,17 @@ def test_get_form(self): """Test get_form().""" # non existent id_string with self.assertRaises(Http404): - get_form({'id_string': 'non_existent_form'}) + get_form({"id_string": "non_existent_form"}) self._publish_transportation_form() # valid xform id_string - kwarg = {'id_string__iexact': self.xform.id_string} + kwarg = {"id_string__iexact": self.xform.id_string} xform = get_form(kwarg) self.assertIsInstance(xform, XForm) # pass a queryset - kwarg['queryset'] = XForm.objects.all() + kwarg["queryset"] = XForm.objects.all() xform = get_form(kwarg) self.assertIsInstance(xform, XForm) @@ -128,36 +132,34 @@ def test_get_form(self): @override_settings(TESTING_MODE=False) def test_get_form_url(self): """Test get_form_url().""" - request = RequestFactory().get('/') + request = RequestFactory().get("/") # default https://ona.io url = get_form_url(request) - self.assertEqual(url, 'https://ona.io') + self.assertEqual(url, "https://ona.io") # with username https://ona.io/bob - url = get_form_url(request, username='bob') - self.assertEqual(url, 'https://ona.io/bob') + url = get_form_url(request, username="bob") + self.assertEqual(url, "https://ona.io/bob") # with http protocol http://ona.io/bob - url = get_form_url(request, username='bob', protocol='http') - self.assertEqual(url, 'http://ona.io/bob') + url = get_form_url(request, username="bob", protocol="http") + self.assertEqual(url, "http://ona.io/bob") # preview url http://ona.io/preview/bob - url = get_form_url( - request, username='bob', protocol='http', preview=True) - self.assertEqual(url, 'http://ona.io/preview/bob') + url = get_form_url(request, username="bob", protocol="http", preview=True) + self.assertEqual(url, "http://ona.io/preview/bob") # with form pk url http://ona.io/bob/1 - url = get_form_url(request, username='bob', xform_pk=1) - self.assertEqual(url, 'https://ona.io/bob/1') + url = get_form_url(request, username="bob", xform_pk=1) + self.assertEqual(url, "https://ona.io/bob/1") # with form uuid url https://ona.io/enketo/492 - url = get_form_url( - request, xform_pk=492, generate_consistent_urls=True) - self.assertEqual(url, 'https://ona.io/enketo/492') + url = get_form_url(request, xform_pk=492, generate_consistent_urls=True) + self.assertEqual(url, "https://ona.io/enketo/492") @override_settings(ZIP_REPORT_ATTACHMENT_LIMIT=8) - @patch('onadata.libs.utils.viewer_tools.report_exception') + @patch("onadata.libs.utils.viewer_tools.report_exception") def test_create_attachments_zipfile_file_too_big(self, rpt_mock): """ When a file is larger than what is allowed in settings an exception @@ -166,16 +168,21 @@ def test_create_attachments_zipfile_file_too_big(self, rpt_mock): self._publish_transportation_form_and_submit_instance() self.media_file = "1335783522563.jpg" media_file = os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', self.surveys[0], self.media_file) + self.this_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + self.media_file, + ) self.instance = Instance.objects.all()[0] self.attachment = Attachment.objects.create( - instance=self.instance, - media_file=File(open(media_file, 'rb'), media_file)) - create_attachments_zipfile(Attachment.objects.all()) + instance=self.instance, media_file=File(open(media_file, "rb"), media_file) + ) + with NamedTemporaryFile() as zip_file: + create_attachments_zipfile(Attachment.objects.all(), zip_file) - message = ( - "Create attachment zip exception", "File is greater than 8 bytes") + message = ("Create attachment zip exception", "File is greater than 8 bytes") self.assertTrue(rpt_mock.called) rpt_mock.assert_called_with(message[0], message[1]) diff --git a/onadata/libs/utils/analytics.py b/onadata/libs/utils/analytics.py index ea57ac72e3..b58e96cfdc 100644 --- a/onadata/libs/utils/analytics.py +++ b/onadata/libs/utils/analytics.py @@ -189,11 +189,11 @@ def track(user, event_name, properties=None, context=None, request=None): if request: context['ip'] = request.META.get('REMOTE_ADDR', '') context['userId'] = user.id - context['receivedAt'] = request.META.get('HTTP_DATE', '') - context['userAgent'] = request.META.get('HTTP_USER_AGENT', '') + context['receivedAt'] = request.headers.get('Date', '') + context['userAgent'] = request.headers.get('User-Agent', '') context['campaign']['source'] = settings.HOSTNAME context['page']['path'] = request.path - context['page']['referrer'] = request.META.get('HTTP_REFERER', '') + context['page']['referrer'] = request.headers.get('Referer', '') context['page']['url'] = request.build_absolute_uri() if _segment: diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index 9a155ad0fb..777833495a 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -6,67 +6,79 @@ import os import sys from datetime import datetime +import six -import httplib2 from celery.backends.rpc import BacklogLimitExceeded from celery.result import AsyncResult from django.conf import settings from django.http import Http404, HttpResponseRedirect from django.shortcuts import get_object_or_404 -from django.utils import six -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from kombu.exceptions import OperationalError -from oauth2client import client as google_client -from oauth2client.client import (HttpAccessTokenRefreshError, - OAuth2WebServerFlow, TokenRevokeError) -from oauth2client.contrib.django_util.storage import \ - DjangoORMStorage as Storage -from requests import ConnectionError from rest_framework import exceptions, status from rest_framework.response import Response from rest_framework.reverse import reverse -from savReaderWriter import SPSSIOError +from google.oauth2.credentials import Credentials + +try: + from savReaderWriter import SPSSIOError +except ImportError: + SPSSIOError = Exception from onadata.apps.main.models import TokenStorageModel from onadata.apps.viewer import tasks as viewer_task from onadata.apps.viewer.models.export import Export, ExportConnectionError -from onadata.libs.exceptions import (J2XException, NoRecordsFoundError, - NoRecordsPermission, ServiceUnavailable) +from onadata.libs.exceptions import ( + J2XException, + NoRecordsFoundError, + NoRecordsPermission, + ServiceUnavailable, +) from onadata.libs.permissions import filter_queryset_xform_meta_perms_sql from onadata.libs.utils import log -from onadata.libs.utils.async_status import (FAILED, PENDING, SUCCESSFUL, - async_status, - celery_state_to_status) -from onadata.libs.utils.common_tags import (DATAVIEW_EXPORT, - GROUPNAME_REMOVED_FLAG, OSM, - SUBMISSION_TIME) +from onadata.libs.utils.google import create_flow +from onadata.libs.utils.async_status import ( + FAILED, + PENDING, + SUCCESSFUL, + async_status, + celery_state_to_status, +) +from onadata.libs.utils.common_tags import ( + DATAVIEW_EXPORT, + GROUPNAME_REMOVED_FLAG, + OSM, + SUBMISSION_TIME, +) from onadata.libs.utils.common_tools import report_exception -from onadata.libs.utils.export_tools import (check_pending_export, - generate_attachments_zip_export, - generate_export, - generate_external_export, - generate_kml_export, - generate_osm_export, - newest_export_for, - parse_request_export_options, - should_create_new_export) +from onadata.libs.utils.export_tools import ( + check_pending_export, + generate_attachments_zip_export, + generate_export, + generate_external_export, + generate_kml_export, + generate_osm_export, + newest_export_for, + parse_request_export_options, + should_create_new_export, +) from onadata.libs.utils.logger_tools import response_with_mimetype_and_name from onadata.libs.utils.model_tools import get_columns_with_hxl # Supported external exports -EXTERNAL_EXPORT_TYPES = ['xls'] +EXTERNAL_EXPORT_TYPES = ["xls"] EXPORT_EXT = { - 'xls': Export.XLS_EXPORT, - 'xlsx': Export.XLS_EXPORT, - 'csv': Export.CSV_EXPORT, - 'csvzip': Export.CSV_ZIP_EXPORT, - 'savzip': Export.SAV_ZIP_EXPORT, - 'uuid': Export.EXTERNAL_EXPORT, - 'kml': Export.KML_EXPORT, - 'zip': Export.ZIP_EXPORT, + "xls": Export.XLS_EXPORT, + "xlsx": Export.XLS_EXPORT, + "csv": Export.CSV_EXPORT, + "csvzip": Export.CSV_ZIP_EXPORT, + "savzip": Export.SAV_ZIP_EXPORT, + "uuid": Export.EXTERNAL_EXPORT, + "kml": Export.KML_EXPORT, + "zip": Export.ZIP_EXPORT, OSM: Export.OSM_EXPORT, - 'gsheets': Export.GOOGLE_SHEETS_EXPORT + "gsheets": Export.GOOGLE_SHEETS_EXPORT, } @@ -89,53 +101,57 @@ def _get_export_type(export_type): export_type = EXPORT_EXT[export_type] else: raise exceptions.ParseError( - _(u"'%(export_type)s' format not known or not implemented!" % - {'export_type': export_type})) + _(f"'{export_type}' format not known or not implemented!") + ) return export_type # pylint: disable=too-many-arguments, too-many-locals, too-many-branches -def custom_response_handler(request, - xform, - query, - export_type, - token=None, - meta=None, - dataview=False, - filename=None): +def custom_response_handler( # noqa: C0901 + request, + xform, + query, + export_type, + token=None, + meta=None, + dataview=False, + filename=None, +): """ Returns a HTTP response with export file for download. """ export_type = _get_export_type(export_type) - if export_type in EXTERNAL_EXPORT_TYPES and \ - (token is not None) or (meta is not None): + if ( + export_type in EXTERNAL_EXPORT_TYPES + and (token is not None) + or (meta is not None) + ): export_type = Export.EXTERNAL_EXPORT options = parse_request_export_options(request.query_params) - dataview_pk = hasattr(dataview, 'pk') and dataview.pk + dataview_pk = hasattr(dataview, "pk") and dataview.pk options["dataview_pk"] = dataview_pk if dataview: - columns_with_hxl = get_columns_with_hxl(xform.survey.get('children')) + columns_with_hxl = get_columns_with_hxl(xform.survey.get("children")) if columns_with_hxl: - options['include_hxl'] = include_hxl_row(dataview.columns, - list(columns_with_hxl)) + options["include_hxl"] = include_hxl_row( + dataview.columns, list(columns_with_hxl) + ) try: - query = filter_queryset_xform_meta_perms_sql(xform, request.user, - query) + query = filter_queryset_xform_meta_perms_sql(xform, request.user, query) except NoRecordsPermission: return Response( - data=json.dumps({ - "details": _("You don't have permission") - }), + data=json.dumps({"details": _("You don't have permission")}), status=status.HTTP_403_FORBIDDEN, - content_type="application/json") + content_type="application/json", + ) if query: - options['query'] = query + options["query"] = query remove_group_name = options.get("remove_group_name") @@ -147,20 +163,21 @@ def custom_response_handler(request, if export_type == Export.GOOGLE_SHEETS_EXPORT: return Response( - data=json.dumps({ - "details": _("Sheets export only supported in async mode") - }), + data=json.dumps( + {"details": _("Sheets export only supported in async mode")} + ), status=status.HTTP_403_FORBIDDEN, - content_type="application/json") + content_type="application/json", + ) # check if we need to re-generate, # we always re-generate if a filter is specified def _new_export(): return _generate_new_export( - request, xform, query, export_type, dataview_pk=dataview_pk) + request, xform, query, export_type, dataview_pk=dataview_pk + ) - if should_create_new_export( - xform, export_type, options, request=request): + if should_create_new_export(xform, export_type, options, request=request): export = _new_export() else: export = newest_export_for(xform, export_type, options) @@ -184,7 +201,8 @@ def _new_export(): show_date = True if filename is None and export.status == Export.SUCCESSFUL: filename = _generate_filename( - request, xform, remove_group_name, dataview_pk=dataview_pk) + request, xform, remove_group_name, dataview_pk=dataview_pk + ) else: show_date = False response = response_with_mimetype_and_name( @@ -192,13 +210,15 @@ def _new_export(): filename, extension=ext, show_date=show_date, - file_path=export.filepath) + file_path=export.filepath, + ) return response -def _generate_new_export(request, xform, query, export_type, - dataview_pk=False): +def _generate_new_export( # noqa: C0901 + request, xform, query, export_type, dataview_pk=False +): query = _set_start_end_params(request, query) extension = _get_extension_from_export_type(export_type) @@ -208,18 +228,17 @@ def _generate_new_export(request, xform, query, export_type, "id_string": xform.id_string, } if query: - options['query'] = query + options["query"] = query options["dataview_pk"] = dataview_pk if export_type == Export.GOOGLE_SHEETS_EXPORT: - options['google_credentials'] = \ - _get_google_credential(request).to_json() + options["google_credentials"] = _get_google_credential(request).to_json() try: if export_type == Export.EXTERNAL_EXPORT: - options['token'] = request.GET.get('token') - options['data_id'] = request.GET.get('data_id') - options['meta'] = request.GET.get('meta') + options["token"] = request.GET.get("token") + options["data_id"] = request.GET.get("data_id") + options["meta"] = request.GET.get("meta") export = generate_external_export( export_type, @@ -227,7 +246,8 @@ def _generate_new_export(request, xform, query, export_type, xform.id_string, None, options, - xform=xform) + xform=xform, + ) elif export_type == Export.OSM_EXPORT: export = generate_osm_export( export_type, @@ -235,7 +255,8 @@ def _generate_new_export(request, xform, query, export_type, xform.id_string, None, options, - xform=xform) + xform=xform, + ) elif export_type == Export.ZIP_EXPORT: export = generate_attachments_zip_export( export_type, @@ -243,7 +264,8 @@ def _generate_new_export(request, xform, query, export_type, xform.id_string, None, options, - xform=xform) + xform=xform, + ) elif export_type == Export.KML_EXPORT: export = generate_kml_export( export_type, @@ -251,7 +273,8 @@ def _generate_new_export(request, xform, query, export_type, xform.id_string, None, options, - xform=xform) + xform=xform, + ) else: options.update(parse_request_export_options(request.query_params)) @@ -259,17 +282,21 @@ def _generate_new_export(request, xform, query, export_type, audit = {"xform": xform.id_string, "export_type": export_type} log.audit_log( - log.Actions.EXPORT_CREATED, request.user, xform.user, - _("Created %(export_type)s export on '%(id_string)s'.") % - {'id_string': xform.id_string, - 'export_type': export_type.upper()}, audit, request) - except NoRecordsFoundError: - raise Http404(_("No records found to export")) + log.Actions.EXPORT_CREATED, + request.user, + xform.user, + _("Created %(export_type)s export on '%(id_string)s'.") + % {"id_string": xform.id_string, "export_type": export_type.upper()}, + audit, + request, + ) + except NoRecordsFoundError as e: + raise Http404(_("No records found to export")) from e except J2XException as e: # j2x exception return async_status(FAILED, str(e)) except SPSSIOError as e: - raise exceptions.ParseError(str(e)) + raise exceptions.ParseError(str(e)) from e else: return export @@ -281,10 +308,13 @@ def log_export(request, xform, export_type): # log download as well audit = {"xform": xform.id_string, "export_type": export_type} log.audit_log( - log.Actions.EXPORT_DOWNLOADED, request.user, xform.user, - _("Downloaded %(export_type)s export on '%(id_string)s'.") % - {'id_string': xform.id_string, - 'export_type': export_type.upper()}, audit, request) + log.Actions.EXPORT_DOWNLOADED, + request.user, + xform.user, + _("Downloaded {export_type.upper()} export on '{id_string}'."), + audit, + request, + ) def external_export_response(export): @@ -292,28 +322,22 @@ def external_export_response(export): Redirects to export_url of XLSReports successful export. In case of a failure, returns a 400 HTTP JSON response with the error message. """ - if isinstance(export, Export) \ - and export.internal_status == Export.SUCCESSFUL: + if isinstance(export, Export) and export.internal_status == Export.SUCCESSFUL: return HttpResponseRedirect(export.export_url) - else: - http_status = status.HTTP_400_BAD_REQUEST + http_status = status.HTTP_400_BAD_REQUEST - return Response( - json.dumps(export), http_status, content_type="application/json") + return Response(json.dumps(export), http_status, content_type="application/json") -def _generate_filename(request, - xform, - remove_group_name=False, - dataview_pk=False): - if request.GET.get('raw'): +def _generate_filename(request, xform, remove_group_name=False, dataview_pk=False): + if request.GET.get("raw"): filename = None else: # append group name removed flag otherwise use the form id_string if remove_group_name: - filename = "{}-{}".format(xform.id_string, GROUPNAME_REMOVED_FLAG) + filename = f"{xform.id_string}-{GROUPNAME_REMOVED_FLAG}" elif dataview_pk: - filename = "{}-{}".format(xform.id_string, DATAVIEW_EXPORT) + filename = f"{xform.id_string}-{DATAVIEW_EXPORT}" else: filename = xform.id_string @@ -322,22 +346,24 @@ def _generate_filename(request, def _set_start_end_params(request, query): # check for start and end params - if 'start' in request.GET or 'end' in request.GET: - query = json.loads(query) \ - if isinstance(query, six.string_types) else query + if "start" in request.GET or "end" in request.GET: + query = json.loads(query) if isinstance(query, six.string_types) else query query[SUBMISSION_TIME] = {} try: - if request.GET.get('start'): - query[SUBMISSION_TIME]['$gte'] = _format_date_for_mongo( - request.GET['start'], datetime) - - if request.GET.get('end'): - query[SUBMISSION_TIME]['$lte'] = _format_date_for_mongo( - request.GET['end'], datetime) - except ValueError: + if request.GET.get("start"): + query[SUBMISSION_TIME]["$gte"] = _format_date_for_mongo( + request.GET["start"] + ) + + if request.GET.get("end"): + query[SUBMISSION_TIME]["$lte"] = _format_date_for_mongo( + request.GET["end"] + ) + except ValueError as e: raise exceptions.ParseError( - _("Dates must be in the format YY_MM_DD_hh_mm_ss")) + _("Dates must be in the format YY_MM_DD_hh_mm_ss") + ) from e else: query = json.dumps(query) @@ -348,16 +374,18 @@ def _get_extension_from_export_type(export_type): extension = export_type if export_type == Export.XLS_EXPORT: - extension = 'xlsx' + extension = "xlsx" elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]: - extension = 'zip' + extension = "zip" return extension -def _format_date_for_mongo(x, datetime): # pylint: disable=W0621, C0103 - return datetime.strptime(x, - '%y_%m_%d_%H_%M_%S').strftime('%Y-%m-%dT%H:%M:%S') +# pylint: disable=invalid-name +def _format_date_for_mongo(datetime_str): + return datetime.strptime(datetime_str, "%y_%m_%d_%H_%M_%S").strftime( + "%Y-%m-%dT%H:%M:%S" + ) def process_async_export(request, xform, export_type, options=None): @@ -388,20 +416,23 @@ def process_async_export(request, xform, export_type, options=None): force_xlsx = options.get("force_xlsx") try: - query = filter_queryset_xform_meta_perms_sql(xform, request.user, - query) + query = filter_queryset_xform_meta_perms_sql(xform, request.user, query) except NoRecordsPermission: payload = {"details": _("You don't have permission")} return Response( data=json.dumps(payload), status=status.HTTP_403_FORBIDDEN, - content_type="application/json") + content_type="application/json", + ) else: if query: - options['query'] = query + options["query"] = query - if export_type in EXTERNAL_EXPORT_TYPES and \ - (token is not None) or (meta is not None): + if ( + export_type in EXTERNAL_EXPORT_TYPES + and (token is not None) + or (meta is not None) + ): export_type = Export.EXTERNAL_EXPORT if export_type == Export.GOOGLE_SHEETS_EXPORT: @@ -409,25 +440,27 @@ def process_async_export(request, xform, export_type, options=None): if isinstance(credential, HttpResponseRedirect): return credential - options['google_credentials'] = credential.to_json() + options["google_credentials"] = credential.to_json() - if should_create_new_export(xform, export_type, options, request=request)\ - or export_type == Export.EXTERNAL_EXPORT: + if ( + should_create_new_export(xform, export_type, options, request=request) + or export_type == Export.EXTERNAL_EXPORT + ): resp = { - u'job_uuid': - _create_export_async( - xform, export_type, query, force_xlsx, options=options) + "job_uuid": _create_export_async( + xform, export_type, query, force_xlsx, options=options + ) } else: - print('Do not create a new export.') + print("Do not create a new export.") export = newest_export_for(xform, export_type, options) if not export.filename: # tends to happen when using newest_export_for. resp = { - u'job_uuid': - _create_export_async( - xform, export_type, query, force_xlsx, options=options) + "job_uuid": _create_export_async( + xform, export_type, query, force_xlsx, options=options + ) } else: resp = export_async_export_response(request, export) @@ -435,21 +468,19 @@ def process_async_export(request, xform, export_type, options=None): return resp -def _create_export_async(xform, - export_type, - query=None, - force_xlsx=False, - options=None): +def _create_export_async( + xform, export_type, query=None, force_xlsx=False, options=None +): + """ + Creates async exports + :param xform: + :param export_type: + :param query: + :param force_xlsx: + :param options: + :return: + job_uuid generated """ - Creates async exports - :param xform: - :param export_type: - :param query: - :param force_xlsx: - :param options: - :return: - job_uuid generated - """ export = check_pending_export(xform, export_type, options) if export: @@ -457,9 +488,10 @@ def _create_export_async(xform, try: export, async_result = viewer_task.create_async_export( - xform, export_type, query, force_xlsx, options=options) - except ExportConnectionError: - raise ServiceUnavailable + xform, export_type, query, force_xlsx, options=options + ) + except ExportConnectionError as e: + raise ServiceUnavailable from e return async_result.task_id @@ -473,14 +505,16 @@ def export_async_export_response(request, export): """ if export.status == Export.SUCCESSFUL: if export.export_type not in [ - Export.EXTERNAL_EXPORT, Export.GOOGLE_SHEETS_EXPORT + Export.EXTERNAL_EXPORT, + Export.GOOGLE_SHEETS_EXPORT, ]: export_url = reverse( - 'export-detail', kwargs={'pk': export.pk}, request=request) + "export-detail", kwargs={"pk": export.pk}, request=request + ) else: export_url = export.export_url resp = async_status(SUCCESSFUL) - resp['export_url'] = export_url + resp["export_url"] = export_url elif export.status == Export.PENDING: resp = async_status(PENDING) else: @@ -493,13 +527,14 @@ def get_async_response(job_uuid, request, xform, count=0): """ Returns the status of an async task for the given job_uuid. """ + def _get_response(): export = get_object_or_404(Export, task_id=job_uuid) return export_async_export_response(request, export) try: job = AsyncResult(job_uuid) - if job.state == 'SUCCESS': + if job.state == "SUCCESS": resp = _get_response() else: resp = async_status(celery_state_to_status(job.state)) @@ -510,33 +545,36 @@ def _get_response(): if isinstance(result, dict): resp.update(result) else: - resp.update({'progress': str(result)}) + resp.update({"progress": str(result)}) except (OperationalError, ConnectionError) as e: report_exception("Connection Error", e, sys.exc_info()) if count > 0: - raise ServiceUnavailable + raise ServiceUnavailable from e return get_async_response(job_uuid, request, xform, count + 1) except BacklogLimitExceeded: # most likely still processing - resp = async_status(celery_state_to_status('PENDING')) + resp = async_status(celery_state_to_status("PENDING")) return resp -def response_for_format(data, format=None): # pylint: disable=W0622 +# pylint: disable=redefined-builtin +def response_for_format(data, format=None): """ Return appropriately formatted data in Response(). """ - if format == 'xml': + if format == "xml": formatted_data = data.xml - elif format == 'xls': + elif format in ("xls", "xlsx"): if not data.xls or not data.xls.storage.exists(data.xls.name): raise Http404() formatted_data = data.xls else: - formatted_data = json.loads(data.json) + formatted_data = ( + json.loads(data.json) if isinstance(data.json, str) else data.json + ) return Response(formatted_data) @@ -544,47 +582,34 @@ def generate_google_web_flow(request): """ Returns a OAuth2WebServerFlow object from the request redirect_uri. """ - if 'redirect_uri' in request.GET: - redirect_uri = request.GET.get('redirect_uri') - elif 'redirect_uri' in request.POST: - redirect_uri = request.POST.get('redirect_uri') - elif 'redirect_uri' in request.query_params: - redirect_uri = request.query_params.get('redirect_uri') - elif 'redirect_uri' in request.data: - redirect_uri = request.data.get('redirect_uri') + if "redirect_uri" in request.GET: + redirect_uri = request.GET.get("redirect_uri") + elif "redirect_uri" in request.POST: + redirect_uri = request.POST.get("redirect_uri") + elif "redirect_uri" in request.query_params: + redirect_uri = request.query_params.get("redirect_uri") + elif "redirect_uri" in request.data: + redirect_uri = request.data.get("redirect_uri") else: redirect_uri = settings.GOOGLE_STEP2_URI - return OAuth2WebServerFlow( - client_id=settings.GOOGLE_OAUTH2_CLIENT_ID, - client_secret=settings.GOOGLE_OAUTH2_CLIENT_SECRET, - scope=' '.join([ - 'https://docs.google.com/feeds/', - 'https://spreadsheets.google.com/feeds/', - 'https://www.googleapis.com/auth/drive.file' - ]), - redirect_uri=redirect_uri, - prompt="consent") + return create_flow(redirect_uri) def _get_google_credential(request): - token = None credential = None if request.user.is_authenticated: - storage = Storage(TokenStorageModel, 'id', request.user, 'credential') - credential = storage.get() - elif request.session.get('access_token'): - credential = google_client.OAuth2Credentials.from_json(token) - - if credential: try: - credential.get_access_token() - except HttpAccessTokenRefreshError: - try: - credential.revoke(httplib2.Http()) - except TokenRevokeError: - storage.delete() - - if not credential or credential.invalid: + storage = TokenStorageModel.objects.get(id=request.user) + credential = storage.credential + except TokenStorageModel.DoesNotExist: + pass + elif request.session.get("access_token"): + credential = Credentials(token=request.session["access_token"]) + + if not credential: google_flow = generate_google_web_flow(request) - return HttpResponseRedirect(google_flow.step1_get_authorize_url()) + authorization_url, _state = google_flow.authorization_url( + access_type="offline", include_granted_scopes="true", prompt="consent" + ) + return HttpResponseRedirect(authorization_url) return credential diff --git a/onadata/libs/utils/audit.py b/onadata/libs/utils/audit.py deleted file mode 100644 index 706750dd2c..0000000000 --- a/onadata/libs/utils/audit.py +++ /dev/null @@ -1 +0,0 @@ -HOME_ACCESSED = "home-accessed" diff --git a/onadata/libs/utils/briefcase_client.py b/onadata/libs/utils/briefcase_client.py index f3f30e576e..5a43c8ad59 100644 --- a/onadata/libs/utils/briefcase_client.py +++ b/onadata/libs/utils/briefcase_client.py @@ -1,36 +1,37 @@ +# -*- coding: utf-8 -*- +"""ODK BriefcaseClient utils module""" import logging import mimetypes import os from io import StringIO from xml.parsers.expat import ExpatError -from future.moves.urllib.parse import urljoin +from six.moves.urllib.parse import urljoin from django.core.files.base import ContentFile from django.core.files.storage import default_storage from django.core.files.uploadedfile import InMemoryUploadedFile from django.db import transaction -from django.utils.translation import ugettext as _ import requests from requests.auth import HTTPDigestAuth from onadata.apps.logger.xform_instance_parser import clean_and_parse_xml from onadata.libs.utils.common_tools import retry -from onadata.libs.utils.logger_tools import (PublishXForm, create_instance, - publish_form) +from onadata.libs.utils.logger_tools import PublishXForm, create_instance, publish_form NUM_RETRIES = 3 def django_file(file_obj, field_name, content_type): + """Return an InMemoryUploadedFile file object.""" return InMemoryUploadedFile( file=file_obj, field_name=field_name, name=file_obj.name, content_type=content_type, size=file_obj.size, - charset=None + charset=None, ) @@ -48,13 +49,13 @@ def _get_form_list(xml_text): xml_doc = clean_and_parse_xml(xml_text) forms = [] - for childNode in xml_doc.childNodes: - if childNode.nodeName == 'xforms': - for xformNode in childNode.childNodes: - if xformNode.nodeName == 'xform': - id_string = node_value(xformNode, 'formID') - download_url = node_value(xformNode, 'downloadUrl') - manifest_url = node_value(xformNode, 'manifestUrl') + for child_node in xml_doc.childNodes: + if child_node.nodeName == "xforms": + for xform_node in child_node.childNodes: + if xform_node.nodeName == "xform": + id_string = node_value(xform_node, "formID") + download_url = node_value(xform_node, "downloadUrl") + manifest_url = node_value(xform_node, "manifestUrl") forms.append((id_string, download_url, manifest_url)) return forms @@ -64,8 +65,8 @@ def _get_instances_uuids(xml_doc): uuids = [] for child_node in xml_doc.childNodes: - if child_node.nodeName == 'idChunk': - for id_node in child_node.getElementsByTagName('id'): + if child_node.nodeName == "idChunk": + for id_node in child_node.getElementsByTagName("id"): if id_node.childNodes: uuid = id_node.childNodes[0].nodeValue uuids.append(uuid) @@ -73,104 +74,112 @@ def _get_instances_uuids(xml_doc): return uuids -class BriefcaseClient(object): +# pylint: disable=too-many-instance-attributes +class BriefcaseClient: + """ODK BriefcaseClient class""" + def __init__(self, url, username, password, user): self.url = url self.user = user self.auth = HTTPDigestAuth(username, password) - self.form_list_url = urljoin(self.url, 'formList') - self.submission_list_url = urljoin(self.url, 'view/submissionList') - self.download_submission_url = urljoin(self.url, - 'view/downloadSubmission') - self.forms_path = os.path.join( - self.user.username, 'briefcase', 'forms') + self.form_list_url = urljoin(self.url, "formList") + self.submission_list_url = urljoin(self.url, "view/submissionList") + self.download_submission_url = urljoin(self.url, "view/downloadSubmission") + self.forms_path = os.path.join(self.user.username, "briefcase", "forms") self.resumption_cursor = 0 - self.logger = logging.getLogger('console_logger') + self.logger = logging.getLogger("console_logger") def download_manifest(self, manifest_url, id_string): + """Downloads the XForm manifest from an ODK server.""" if self._get_response(manifest_url): - manifest_res = self._current_response + manifest_res = getattr(self, "_current_response") try: manifest_doc = clean_and_parse_xml(manifest_res.content) except ExpatError: return - manifest_path = os.path.join( - self.forms_path, id_string, 'form-media') - self.logger.debug("Downloading media files for %s" % id_string) + manifest_path = os.path.join(self.forms_path, id_string, "form-media") + self.logger.debug("Downloading media files for %s", id_string) self.download_media_files(manifest_doc, manifest_path) def download_xforms(self, include_instances=False): + """Downloads the XForm XML form an ODK server.""" # fetch formList if not self._get_response(self.form_list_url): - response = self._current_response.content \ - if self._current_response else "Unknown Error" - self.logger.error("Failed to download xforms %s." % response) + _current_response = getattr(self, "_current_response") + response = ( + _current_response.content if _current_response else "Unknown Error" + ) + self.logger.error("Failed to download xforms %s.", response) return - response = self._current_response + response = getattr(self, "_current_response") forms = _get_form_list(response.content) - self.logger.debug('Successfull fetched %s.' % self.form_list_url) + self.logger.debug("Successfull fetched %s.", self.form_list_url) for id_string, download_url, manifest_url in forms: - form_path = os.path.join( - self.forms_path, id_string, '%s.xml' % id_string) + form_path = os.path.join(self.forms_path, id_string, f"{id_string}.xml") if not default_storage.exists(form_path): if not self._get_response(download_url): - self.logger.error("Failed to download xform %s." - % download_url) + self.logger.error("Failed to download xform %s.", download_url) continue - form_res = self._current_response + form_res = getattr(self, "_current_response") content = ContentFile(form_res.content.strip()) default_storage.save(form_path, content) else: form_res = default_storage.open(form_path) content = form_res.read() - self.logger.debug("Fetched %s." % download_url) + self.logger.debug("Fetched %s.", download_url) if manifest_url: self.download_manifest(manifest_url, id_string) if include_instances: self.download_instances(id_string) - self.logger.debug("Done downloading submissions for %s" % - id_string) + self.logger.debug("Done downloading submissions for %s", id_string) @retry(NUM_RETRIES) def _get_response(self, url, params=None): - self._current_response = None + """ + Downloads the url and sets self._current_response with the contents. + """ + setattr(self, "_current_response", None) response = requests.get(url, auth=self.auth, params=params) success = response.status_code == 200 - self._current_response = response + setattr(self, "_current_response", response) return success @retry(NUM_RETRIES) def _get_media_response(self, url): - self._current_response = None + """ + Downloads the media file and sets self._current_response with the contents. + """ + setattr(self, "_current_response", None) head_response = requests.head(url, auth=self.auth) # S3 redirects, avoid using formhub digest on S3 if head_response.status_code == 302: - url = head_response.headers.get('location') + url = head_response.headers.get("location") response = requests.get(url) success = response.status_code == 200 - self._current_response = response + setattr(self, "_current_response", response) return success def download_media_files(self, xml_doc, media_path): - for media_node in xml_doc.getElementsByTagName('mediaFile'): - filename_node = media_node.getElementsByTagName('filename') - url_node = media_node.getElementsByTagName('downloadUrl') + """Downloads media files from an ODK server.""" + for media_node in xml_doc.getElementsByTagName("mediaFile"): + filename_node = media_node.getElementsByTagName("filename") + url_node = media_node.getElementsByTagName("downloadUrl") if filename_node and url_node: filename = filename_node[0].childNodes[0].nodeValue path = os.path.join(media_path, filename) @@ -178,47 +187,59 @@ def download_media_files(self, xml_doc, media_path): continue download_url = url_node[0].childNodes[0].nodeValue if self._get_media_response(download_url): - download_res = self._current_response + download_res = getattr(self, "_current_response") media_content = ContentFile(download_res.content) default_storage.save(path, media_content) - self.logger.debug("Fetched %s." % filename) + self.logger.debug("Fetched %s.", filename) else: - self.logger.error("Failed to fetch %s." % filename) + self.logger.error("Failed to fetch %s.", filename) + # pylint: disable=too-many-locals def download_instances(self, form_id, cursor=0, num_entries=100): - self.logger.debug("Starting submissions download for %s" % form_id) - if not self._get_response(self.submission_list_url, - params={'formId': form_id, - 'numEntries': num_entries, - 'cursor': cursor}): - self.logger.error("Fetching %s formId: %s, cursor: %s" % - (self.submission_list_url, form_id, cursor)) + """Download the XML submissions.""" + self.logger.debug("Starting submissions download for %s", form_id) + downloaded = self._get_response( + self.submission_list_url, + params={"formId": form_id, "numEntries": num_entries, "cursor": cursor}, + ) + if not downloaded: + self.logger.error( + "Fetching %s formId: %s, cursor: %s", + self.submission_list_url, + form_id, + cursor, + ) return - response = self._current_response - self.logger.debug("Fetching %s formId: %s, cursor: %s" % - (self.submission_list_url, form_id, cursor)) + response = getattr(self, "_current_response") + self.logger.debug( + "Fetching %s formId: %s, cursor: %s", + self.submission_list_url, + form_id, + cursor, + ) try: xml_doc = clean_and_parse_xml(response.content) except ExpatError: return instances = _get_instances_uuids(xml_doc) - path = os.path.join(self.forms_path, form_id, 'instances') + path = os.path.join(self.forms_path, form_id, "instances") for uuid in instances: - self.logger.debug("Fetching %s %s submission" % (uuid, form_id)) - form_str = u'%(formId)s[@version=null and @uiVersion=null]/'\ - u'%(formId)s[@key=%(instanceId)s]' % { - 'formId': form_id, - 'instanceId': uuid - } - instance_path = os.path.join(path, uuid.replace(':', ''), - 'submission.xml') + self.logger.debug("Fetching %s %s submission", uuid, form_id) + form_str = ( + "%(formId)s[@version=null and @uiVersion=null]/" + "%(formId)s[@key=%(instanceId)s]" + % {"formId": form_id, "instanceId": uuid} + ) + instance_path = os.path.join(path, uuid.replace(":", ""), "submission.xml") if not default_storage.exists(instance_path): - if self._get_response(self.download_submission_url, - params={'formId': form_str}): - instance_res = self._current_response + downloaded = self._get_response( + self.download_submission_url, params={"formId": form_str} + ) + if downloaded: + instance_res = getattr(self, "_current_response") content = instance_res.content.strip() default_storage.save(instance_path, ContentFile(content)) else: @@ -232,12 +253,12 @@ def download_instances(self, form_id, cursor=0, num_entries=100): except ExpatError: continue - media_path = os.path.join(path, uuid.replace(':', '')) + media_path = os.path.join(path, uuid.replace(":", "")) self.download_media_files(instance_doc, media_path) - self.logger.debug("Fetched %s %s submission" % (form_id, uuid)) + self.logger.debug("Fetched %s %s submission", form_id, uuid) - if xml_doc.getElementsByTagName('resumptionCursor'): - rs_node = xml_doc.getElementsByTagName('resumptionCursor')[0] + if xml_doc.getElementsByTagName("resumptionCursor"): + rs_node = xml_doc.getElementsByTagName("resumptionCursor")[0] cursor = rs_node.childNodes[0].nodeValue if self.resumption_cursor != cursor: @@ -246,6 +267,7 @@ def download_instances(self, form_id, cursor=0, num_entries=100): @transaction.atomic def _upload_xform(self, path, file_name): + """Publishes the XForm XML to the XForm model.""" xml_file = default_storage.open(path) xml_file.name = file_name k = PublishXForm(xml_file, self.user) @@ -253,40 +275,45 @@ def _upload_xform(self, path, file_name): return publish_form(k.publish_xform) def _upload_instance(self, xml_file, instance_dir_path, files): + """ + Adds an xform submission to the Instance model. + """ xml_doc = clean_and_parse_xml(xml_file.read()) xml = StringIO() de_node = xml_doc.documentElement for node in de_node.firstChild.childNodes: xml.write(node.toxml()) - new_xml_file = ContentFile(xml.getvalue().encode('utf-8')) - new_xml_file.content_type = 'text/xml' + new_xml_file = ContentFile(xml.getvalue().encode("utf-8")) + new_xml_file.content_type = "text/xml" xml.close() attachments = [] - for attach in de_node.getElementsByTagName('mediaFile'): - filename_node = attach.getElementsByTagName('filename') + for attach in de_node.getElementsByTagName("mediaFile"): + filename_node = attach.getElementsByTagName("filename") filename = filename_node[0].childNodes[0].nodeValue if filename in files: file_obj = default_storage.open( - os.path.join(instance_dir_path, filename)) - mimetype, encoding = mimetypes.guess_type(file_obj.name) - media_obj = django_file(file_obj, 'media_files[]', mimetype) + os.path.join(instance_dir_path, filename) + ) + mimetype, _encoding = mimetypes.guess_type(file_obj.name) + media_obj = django_file(file_obj, "media_files[]", mimetype) attachments.append(media_obj) create_instance(self.user.username, new_xml_file, attachments) def _upload_instances(self, path): instances_count = 0 - dirs, not_in_use = default_storage.listdir(path) + dirs, _not_in_use = default_storage.listdir(path) for instance_dir in dirs: instance_dir_path = os.path.join(path, instance_dir) - i_dirs, files = default_storage.listdir(instance_dir_path) + _dirs, files = default_storage.listdir(instance_dir_path) xml_file = None - if 'submission.xml' in files: + if "submission.xml" in files: file_obj = default_storage.open( - os.path.join(instance_dir_path, 'submission.xml')) + os.path.join(instance_dir_path, "submission.xml") + ) xml_file = file_obj if xml_file: @@ -294,30 +321,40 @@ def _upload_instances(self, path): self._upload_instance(xml_file, instance_dir_path, files) except ExpatError: continue + # pylint: disable=broad-except except Exception as e: - logging.exception(_( - u'Ignoring exception, processing XML submission ' - 'raised exception: %s' % str(e))) + # keep going despite some errors. + logging.exception( + ( + "Ignoring exception, processing XML submission " + "raised exception: %s" + ), + str(e), + ) else: instances_count += 1 return instances_count def push(self): - dirs, files = default_storage.listdir(self.forms_path) + """Publishes XForms and XForm submissions.""" + dirs, _files = default_storage.listdir(self.forms_path) for form_dir in dirs: dir_path = os.path.join(self.forms_path, form_dir) form_dirs, form_files = default_storage.listdir(dir_path) - form_xml = '%s.xml' % form_dir + form_xml = f"{form_dir}.xml" if form_xml in form_files: form_xml_path = os.path.join(dir_path, form_xml) - x = self._upload_xform(form_xml_path, form_xml) - if isinstance(x, dict): - self.logger.error("Failed to publish %s" % form_dir) + published_xform = self._upload_xform(form_xml_path, form_xml) + if isinstance(published_xform, dict): + self.logger.error("Failed to publish %s", form_dir) else: - self.logger.debug("Successfully published %s" % form_dir) - if 'instances' in form_dirs: + self.logger.debug("Successfully published %s", form_dir) + if "instances" in form_dirs: self.logger.debug("Uploading instances") - c = self._upload_instances(os.path.join(dir_path, 'instances')) - self.logger.debug("Published %d instances for %s" % - (c, form_dir)) + submission_count = self._upload_instances( + os.path.join(dir_path, "instances") + ) + self.logger.debug( + "Published %d instances for %s", submission_count, form_dir + ) diff --git a/onadata/libs/utils/chart_tools.py b/onadata/libs/utils/chart_tools.py index a411f4ae11..a26796c627 100644 --- a/onadata/libs/utils/chart_tools.py +++ b/onadata/libs/utils/chart_tools.py @@ -1,53 +1,61 @@ +# -*- coding=utf-8 -*- +""" +Chart utility functions. +""" from __future__ import unicode_literals import re -import six - -from builtins import str as text from collections import OrderedDict -from past.builtins import basestring from django.db.utils import DataError from django.http import Http404 + +import six from rest_framework.exceptions import ParseError from onadata.apps.logger.models.data_view import DataView from onadata.apps.logger.models.xform import XForm -from onadata.libs.data.query import \ - get_form_submissions_aggregated_by_select_one -from onadata.libs.data.query import get_form_submissions_grouped_by_field -from onadata.libs.data.query import get_form_submissions_grouped_by_select_one +from onadata.libs.data.query import ( + get_form_submissions_aggregated_by_select_one, + get_form_submissions_grouped_by_field, + get_form_submissions_grouped_by_select_one, +) from onadata.libs.utils import common_tags # list of fields we can chart CHART_FIELDS = [ - 'select one', 'integer', 'decimal', 'date', 'datetime', 'start', 'end', - 'today' + "select one", + "integer", + "decimal", + "date", + "datetime", + "start", + "end", + "today", ] # numeric, categorized DATA_TYPE_MAP = { - 'integer': 'numeric', - 'decimal': 'numeric', - 'datetime': 'time_based', - 'date': 'time_based', - 'start': 'time_based', - 'end': 'time_based', - 'today': 'time_based', - 'calculate': 'numeric', + "integer": "numeric", + "decimal": "numeric", + "datetime": "time_based", + "date": "time_based", + "start": "time_based", + "end": "time_based", + "today": "time_based", + "calculate": "numeric", } FIELD_DATA_MAP = { - common_tags.SUBMISSION_TIME: - ('Submission Time', '_submission_time', 'datetime'), - common_tags.SUBMITTED_BY: ('Submission By', '_submitted_by', 'text'), - common_tags.DURATION: ('Duration', '_duration', 'integer') + common_tags.SUBMISSION_TIME: ("Submission Time", "_submission_time", "datetime"), + common_tags.SUBMITTED_BY: ("Submission By", "_submitted_by", "text"), + common_tags.DURATION: ("Duration", "_duration", "integer"), } CHARTS_PER_PAGE = 20 POSTGRES_ALIAS_LENGTH = 63 -timezone_re = re.compile(r'(.+)\+(\d+)') +timezone_re = re.compile(r"(.+)\+(\d+)") def utc_time_string_for_javascript(date_string): @@ -62,23 +70,25 @@ def utc_time_string_for_javascript(date_string): match = timezone_re.match(date_string) if not match: raise ValueError( - "{} fos not match the format 2014-01-16T12:07:23.322+03".format( - date_string)) + f"{date_string} fos not match the format 2014-01-16T12:07:23.322+03" + ) date_time = match.groups()[0] - tz = match.groups()[1] - if len(tz) == 2: - tz += '00' - elif len(tz) != 4: - raise ValueError("len of {} must either be 2 or 4") + timezone = match.groups()[1] + if len(timezone) == 2: + timezone += "00" + elif len(timezone) != 4: + raise ValueError(f"len of {timezone} must either be 2 or 4") - return "{}+{}".format(date_time, tz) + return f"{date_time}+{timezone}" def find_choice_label(choices, string): + """Returns the choice label of the given ``string``.""" for choice in choices: - if choice['name'] == string: - return choice['label'] + if choice["name"] == string: + return choice["label"] + return None def get_field_choices(field, xform): @@ -88,13 +98,13 @@ def get_field_choices(field, xform): :param xform: :return: Form field choices """ - choices = xform.survey.get('choices') + choices = xform.survey.get("choices") - if isinstance(field, basestring): + if isinstance(field, str): choices = choices.get(field) - elif 'name' in field and field.name in choices: + elif "name" in field and field.name in choices: choices = choices.get(field.name) - elif 'itemset' in field: + elif "itemset" in field: choices = choices.get(field.itemset) return choices @@ -120,9 +130,7 @@ def get_choice_label(choices, string): labels.append(label) else: # Try to get labels by splitting the string - labels = [ - find_choice_label(choices, name) for name in string.split(" ") - ] + labels = [find_choice_label(choices, name) for name in string.split(" ")] # If any split string does not have a label it is not a multiselect # but a missing label, use string @@ -141,34 +149,29 @@ def _flatten_multiple_dict_into_one(field_name, group_by_name, data): # truncate field name to 63 characters to fix #354 truncated_field_name = field_name[0:POSTGRES_ALIAS_LENGTH] truncated_group_by_name = group_by_name[0:POSTGRES_ALIAS_LENGTH] - final = [{ - truncated_field_name: b, - 'items': [] - } for b in list({a.get(truncated_field_name) - for a in data})] - - for a in data: - for b in final: - if a.get(truncated_field_name) == b.get(truncated_field_name): - b['items'].append({ - truncated_group_by_name: - a.get(truncated_group_by_name), - 'count': - a.get('count') - }) + final = [ + {truncated_field_name: b, "items": []} + for b in list({a.get(truncated_field_name) for a in data}) + ] + + for round_1 in data: + for round_2 in final: + if round_1.get(truncated_field_name) == round_2.get(truncated_field_name): + round_2["items"].append( + { + truncated_group_by_name: round_1.get(truncated_group_by_name), + "count": round_1.get("count"), + } + ) return final -def _use_labels_from_field_name(field_name, - field, - data_type, - data, - choices=None): +def _use_labels_from_field_name(field_name, field, data_type, data, choices=None): # truncate field name to 63 characters to fix #354 truncated_name = field_name[0:POSTGRES_ALIAS_LENGTH] - if data_type == 'categorized' and field_name != common_tags.SUBMITTED_BY: + if data_type == "categorized" and field_name != common_tags.SUBMITTED_BY: if data: if field.children: choices = field.children @@ -176,60 +179,58 @@ def _use_labels_from_field_name(field_name, for item in data: if truncated_name in item: item[truncated_name] = get_choice_label( - choices, item[truncated_name]) + choices, item[truncated_name] + ) for item in data: if field_name != truncated_name: item[field_name] = item[truncated_name] - del (item[truncated_name]) + del item[truncated_name] return data -def _use_labels_from_group_by_name(field_name, - field, - data_type, - data, - choices=None): +def _use_labels_from_group_by_name( # noqa C901 + field_name, field, data_type, data, choices=None +): # truncate field name to 63 characters to fix #354 truncated_name = field_name[0:POSTGRES_ALIAS_LENGTH] - if data_type == 'categorized': + if data_type == "categorized": if data: if field.children: choices = field.children for item in data: - if 'items' in item: - for i in item.get('items'): - i[truncated_name] = get_choice_label(choices, - i[truncated_name]) + if "items" in item: + for i in item.get("items"): + i[truncated_name] = get_choice_label(choices, i[truncated_name]) else: - item[truncated_name] = \ - get_choice_label(choices, item[truncated_name]) + item[truncated_name] = get_choice_label( + choices, item[truncated_name] + ) for item in data: - if 'items' in item: - for i in item.get('items'): + if "items" in item: + for i in item.get("items"): if field_name != truncated_name: i[field_name] = i[truncated_name] - del (i[truncated_name]) + del i[truncated_name] else: if field_name != truncated_name: item[field_name] = item[truncated_name] - del (item[truncated_name]) + del item[truncated_name] return data -def build_chart_data_for_field(xform, - field, - language_index=0, - choices=None, - group_by=None, - data_view=None): +# pylint: disable=too-many-locals,too-many-branches,too-many-arguments +def build_chart_data_for_field( # noqa C901 + xform, field, language_index=0, choices=None, group_by=None, data_view=None +): + """Returns the chart data for a given field.""" # check if its the special _submission_time META - if isinstance(field, basestring): + if isinstance(field, str): field_label, field_xpath, field_type = FIELD_DATA_MAP.get(field) else: # TODO: merge choices with results and set 0's on any missing fields, @@ -239,96 +240,106 @@ def build_chart_data_for_field(xform, field_xpath = field.get_abbreviated_xpath() field_type = field.type - data_type = DATA_TYPE_MAP.get(field_type, 'categorized') - field_name = field.name if not isinstance(field, basestring) else field + data_type = DATA_TYPE_MAP.get(field_type, "categorized") + field_name = field.name if not isinstance(field, str) else field if group_by and isinstance(group_by, list): group_by_name = [ - g.get_abbreviated_xpath() if not isinstance(g, basestring) else g - for g in group_by + g.get_abbreviated_xpath() if not isinstance(g, str) else g for g in group_by ] result = get_form_submissions_aggregated_by_select_one( - xform, field_xpath, field_name, group_by_name, data_view) + xform, field_xpath, field_name, group_by_name, data_view + ) elif group_by: - group_by_name = group_by.get_abbreviated_xpath() \ - if not isinstance(group_by, basestring) else group_by - - if (field_type == common_tags.SELECT_ONE or - field_name == common_tags.SUBMITTED_BY) and \ - isinstance(group_by, six.string_types): + group_by_name = ( + group_by.get_abbreviated_xpath() + if not isinstance(group_by, str) + else group_by + ) + + if ( + field_type == common_tags.SELECT_ONE + or field_name == common_tags.SUBMITTED_BY + ) and isinstance(group_by, six.string_types): result = get_form_submissions_grouped_by_select_one( - xform, field_xpath, group_by_name, field_name, data_view) - elif field_type in common_tags.NUMERIC_LIST and \ - isinstance(group_by, six.string_types): + xform, field_xpath, group_by_name, field_name, data_view + ) + elif field_type in common_tags.NUMERIC_LIST and isinstance( + group_by, six.string_types + ): result = get_form_submissions_aggregated_by_select_one( - xform, field_xpath, field_name, group_by_name, data_view) - elif (field_type == common_tags.SELECT_ONE or - field_name == common_tags.SUBMITTED_BY) and \ - group_by.type == common_tags.SELECT_ONE: + xform, field_xpath, field_name, group_by_name, data_view + ) + elif ( + field_type == common_tags.SELECT_ONE + or field_name == common_tags.SUBMITTED_BY + ) and group_by.type == common_tags.SELECT_ONE: result = get_form_submissions_grouped_by_select_one( - xform, field_xpath, group_by_name, field_name, data_view) - - result = _flatten_multiple_dict_into_one(field_name, group_by_name, - result) - elif field_type in common_tags.NUMERIC_LIST \ - and group_by.type == common_tags.SELECT_ONE: + xform, field_xpath, group_by_name, field_name, data_view + ) + + result = _flatten_multiple_dict_into_one(field_name, group_by_name, result) + elif ( + field_type in common_tags.NUMERIC_LIST + and group_by.type == common_tags.SELECT_ONE + ): result = get_form_submissions_aggregated_by_select_one( - xform, field_xpath, field_name, group_by_name, data_view) + xform, field_xpath, field_name, group_by_name, data_view + ) else: - raise ParseError('Cannot group by %s' % group_by_name) + raise ParseError(f"Cannot group by {group_by_name}") else: - result = get_form_submissions_grouped_by_field(xform, field_xpath, - field_name, data_view) + result = get_form_submissions_grouped_by_field( + xform, field_xpath, field_name, data_view + ) result = _use_labels_from_field_name( - field_name, field, data_type, result, choices=choices) + field_name, field, data_type, result, choices=choices + ) - if group_by and not isinstance(group_by, six.string_types + (list, )): - group_by_data_type = DATA_TYPE_MAP.get(group_by.type, 'categorized') + if group_by and not isinstance(group_by, six.string_types + (list,)): + group_by_data_type = DATA_TYPE_MAP.get(group_by.type, "categorized") grp_choices = get_field_choices(group_by, xform) result = _use_labels_from_group_by_name( - group_by_name, - group_by, - group_by_data_type, - result, - choices=grp_choices) + group_by_name, group_by, group_by_data_type, result, choices=grp_choices + ) elif group_by and isinstance(group_by, list): - for g in group_by: - if isinstance(g, six.string_types): + for a_group in group_by: + if isinstance(a_group, six.string_types): continue - group_by_data_type = DATA_TYPE_MAP.get(g.type, 'categorized') - grp_choices = get_field_choices(g, xform) + group_by_data_type = DATA_TYPE_MAP.get(a_group.type, "categorized") + grp_choices = get_field_choices(a_group, xform) result = _use_labels_from_group_by_name( - g.get_abbreviated_xpath(), - g, + a_group.get_abbreviated_xpath(), + a_group, group_by_data_type, result, - choices=grp_choices) + choices=grp_choices, + ) if not group_by: - result = sorted(result, key=lambda d: d['count']) + result = sorted(result, key=lambda d: d["count"]) # for date fields, strip out None values - if data_type == 'time_based': + if data_type == "time_based": result = [r for r in result if r.get(field_name) is not None] # for each check if it matches the timezone regexp and convert for js - for r in result: - if timezone_re.match(r[field_name]): + for row in result: + if timezone_re.match(row[field_name]): try: - r[field_name] = utc_time_string_for_javascript( - r[field_name]) + row[field_name] = utc_time_string_for_javascript(row[field_name]) except ValueError: pass return { - 'data': result, - 'data_type': data_type, - 'field_label': field_label, - 'field_xpath': field_xpath, - 'field_name': field_name, - 'field_type': field_type, - 'grouped_by': group_by_name if group_by else None + "data": result, + "data_type": data_type, + "field_label": field_label, + "field_xpath": field_xpath, + "field_name": field_name, + "field_type": field_type, + "grouped_by": group_by_name if group_by else None, } @@ -343,6 +354,7 @@ def calculate_ranges(page, items_per_page, total_items): def build_chart_data(xform, language_index=0, page=0): + """Returns chart data for all the fields in the ``xform``.""" # only use chart-able fields fields = [e for e in xform.survey_elements if e.type in CHART_FIELDS] @@ -355,12 +367,12 @@ def build_chart_data(xform, language_index=0, page=0): fields = fields[start:end] return [ - build_chart_data_for_field(xform, field, language_index) - for field in fields + build_chart_data_for_field(xform, field, language_index) for field in fields ] def build_chart_data_from_widget(widget, language_index=0): + """Returns chart data from a widget.""" if isinstance(widget.content_object, XForm): xform = widget.content_object @@ -379,19 +391,17 @@ def build_chart_data_from_widget(widget, language_index=0): fields = [e for e in xform.survey_elements if e.name == field_name] if len(fields) == 0: - raise ParseError("Field %s does not not exist on the form" % - field_name) + raise ParseError(f"Field {field_name} does not not exist on the form") field = fields[0] - choices = xform.survey.get('choices') + choices = xform.survey.get("choices") if choices: choices = choices.get(field_name) try: - data = build_chart_data_for_field( - xform, field, language_index, choices=choices) + data = build_chart_data_for_field(xform, field, language_index, choices=choices) except DataError as e: - raise ParseError(text(e)) + raise ParseError(str(e)) from e return data @@ -408,22 +418,25 @@ def _get_field_from_field_fn(field_str, xform, field_fn): # use specified field to get summary fields = [e for e in xform.survey_elements if field_fn(e) == field_str] if len(fields) == 0: - raise Http404("Field %s does not not exist on the form" % - field_str) + raise Http404(f"Field {field_str} does not not exist on the form") field = fields[0] return field def get_field_from_field_name(field_name, xform): + """Returns the field if the ``field_name`` is in the ``xform``.""" return _get_field_from_field_fn(field_name, xform, lambda x: x.name) def get_field_from_field_xpath(field_xpath, xform): + """Returns the field if the ``field_xpath`` is in the ``xform``.""" return _get_field_from_field_fn( - field_xpath, xform, lambda x: x.get_abbreviated_xpath()) + field_xpath, xform, lambda x: x.get_abbreviated_xpath() + ) def get_field_label(field, language_index=0): + """Returns the ``field``'s label or name based on selected ``language_index``.'""" # check if label is dict i.e. multilang if isinstance(field.label, dict) and len(list(field.label)) > 0: languages = list(OrderedDict(field.label)) @@ -435,12 +448,10 @@ def get_field_label(field, language_index=0): return field_label -def get_chart_data_for_field(field_name, - xform, - accepted_format, - group_by, - field_xpath=None, - data_view=None): +# pylint: disable=too-many-arguments +def get_chart_data_for_field( # noqa C901 + field_name, xform, accepted_format, group_by, field_xpath=None, data_view=None +): """ Get chart data for a given xlsform field. """ @@ -450,10 +461,9 @@ def get_chart_data_for_field(field_name, field = get_field_from_field_name(field_name, xform) if group_by: - if len(group_by.split(',')) > 1: + if len(group_by.split(",")) > 1: group_by = [ - get_field_from_field_xpath(g, xform) - for g in group_by.split(',') + get_field_from_field_xpath(g, xform) for g in group_by.split(",") ] else: group_by = get_field_from_field_xpath(group_by, xform) @@ -465,21 +475,18 @@ def get_chart_data_for_field(field_name, try: data = build_chart_data_for_field( - xform, - field, - choices=choices, - group_by=group_by, - data_view=data_view) + xform, field, choices=choices, group_by=group_by, data_view=data_view + ) except DataError as e: - raise ParseError(text(e)) + raise ParseError(str(e)) from e else: - if accepted_format == 'json' or not accepted_format: + if accepted_format == "json" or not accepted_format: xform = xform.pk - elif accepted_format == 'html' and 'data' in data: - for item in data['data']: + elif accepted_format == "html" and "data" in data: + for item in data["data"]: if isinstance(item[field_name], list): - item[field_name] = ', '.join(item[field_name]) + item[field_name] = ", ".join(item[field_name]) - data.update({'xform': xform}) + data.update({"xform": xform}) return data diff --git a/onadata/libs/utils/common_tags.py b/onadata/libs/utils/common_tags.py index dab0426999..944b865dff 100644 --- a/onadata/libs/utils/common_tags.py +++ b/onadata/libs/utils/common_tags.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ # WE SHOULD PUT MORE STRUCTURE ON THESE TAGS SO WE CAN ACCESS DOCUMENT # FIELDS ELEGANTLY diff --git a/onadata/libs/utils/common_tools.py b/onadata/libs/utils/common_tools.py index 4abfece1e0..8c03242cf3 100644 --- a/onadata/libs/utils/common_tools.py +++ b/onadata/libs/utils/common_tools.py @@ -11,17 +11,16 @@ import traceback import uuid from io import BytesIO -from past.builtins import basestring from django.conf import settings from django.core.mail import mail_admins from django.db import OperationalError -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ import six from raven.contrib.django.raven_compat.models import client -TRUE_VALUES = ['TRUE', 'T', '1', 1] +TRUE_VALUES = ["TRUE", "T", "1", 1] def str_to_bool(str_var): @@ -41,17 +40,16 @@ def get_boolean_value(str_var, default=None): """ Converts a string into boolean """ - if isinstance(str_var, basestring) and \ - str_var.lower() in ['true', 'false']: + if isinstance(str_var, str) and str_var.lower() in ["true", "false"]: return str_to_bool(str_var) return str_var if default else False def get_uuid(hex_only: bool = True): - ''' + """ Return UUID4 hex value - ''' + """ return uuid.uuid4().hex if hex_only else str(uuid.uuid4()) @@ -61,26 +59,24 @@ def report_exception(subject, info, exc_info=None): testing sends email to mail_admins. """ # Add hostname to subject mail - subject = "{0} - {1}".format(subject, settings.HOSTNAME) + subject = f"{subject} - {settings.HOSTNAME}" if exc_info: cls, err = exc_info[:2] - message = _(u"Exception in request:" - u" %(class)s: %(error)s")\ - % {'class': cls.__name__, 'error': err} - message += u"".join(traceback.format_exception(*exc_info)) + message = _(f"Exception in request: {cls.__name__}: {err}") + message += "".join(traceback.format_exception(*exc_info)) # send to sentry try: client.captureException(exc_info) except Exception: # pylint: disable=broad-except - logging.exception(_(u'Sending to Sentry failed.')) + logging.exception(_("Sending to Sentry failed.")) else: - message = u"%s" % info + message = f"{info}" if settings.DEBUG or settings.TESTING_MODE: - sys.stdout.write("Subject: %s\n" % subject) - sys.stdout.write("Message: %s\n" % message) + sys.stdout.write(f"Subject: {subject}\n") + sys.stdout.write(f"Message: {message}\n") else: mail_admins(subject=subject, message=message) @@ -89,12 +85,12 @@ def filename_from_disposition(content_disposition): """ Gets a filename from the given content disposition header. """ - filename_pos = content_disposition.index('filename=') + filename_pos = content_disposition.index("filename=") if filename_pos == -1: raise Exception('"filename=" not found in content disposition file') - return content_disposition[filename_pos + len('filename='):] + return content_disposition[filename_pos + len("filename=") :] def get_response_content(response, decode=True): @@ -106,7 +102,7 @@ def get_response_content(response, decode=True): :param response: The response to extract content from. :param decode: If true decode as utf-8, default True. """ - contents = '' + contents = "" if response.streaming: actual_content = BytesIO() for content in response.streaming_content: @@ -117,16 +113,15 @@ def get_response_content(response, decode=True): contents = response.content if decode: - return contents.decode('utf-8') - else: - return contents + return contents.decode("utf-8") + return contents def json_stream(data, json_string): """ Generator function to stream JSON data """ - yield '[' + yield "[" try: data = data.__iter__() item = next(data) @@ -134,7 +129,7 @@ def json_stream(data, json_string): try: next_item = next(data) yield json_string(item) - yield ',' + yield "," item = next_item except StopIteration: yield json_string(item) @@ -142,7 +137,7 @@ def json_stream(data, json_string): except (AttributeError, StopIteration): pass finally: - yield ']' + yield "]" def retry(tries, delay=3, backoff=2): @@ -181,8 +176,10 @@ def function_retry(self, *args, **kwargs): else: return result # Last ditch effort run against master database - if len(getattr(settings, 'SLAVE_DATABASES', [])): + if len(getattr(settings, "SLAVE_DATABASES", [])): + # pylint: disable=import-outside-toplevel from multidb.pinning import use_master + with use_master: return func(self, *args, **kwargs) @@ -190,11 +187,12 @@ def function_retry(self, *args, **kwargs): return func(self, *args, **kwargs) return function_retry + return decorator_retry def merge_dicts(*dict_args): - """ Given any number of dicts, shallow copy and merge into a new dict, + """Given any number of dicts, shallow copy and merge into a new dict, precedence goes to key value pairs in latter dicts. """ result = {} @@ -206,9 +204,11 @@ def merge_dicts(*dict_args): def cmp_to_key(mycmp): - """ Convert a cmp= function into a key= function - """ - class K(object): + """Convert a cmp= function into a key= function""" + + class ComparatorClass: + """A class that implements comparison methods.""" + def __init__(self, obj, *args): self.obj = obj @@ -229,4 +229,5 @@ def __ge__(self, other): def __ne__(self, other): return mycmp(self.obj, other.obj) != 0 - return K + + return ComparatorClass diff --git a/onadata/libs/utils/country_field.py b/onadata/libs/utils/country_field.py index ee48e1ab67..6974a6cf23 100644 --- a/onadata/libs/utils/country_field.py +++ b/onadata/libs/utils/country_field.py @@ -1,5 +1,5 @@ from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ # http://www.unece.org/cefact/locode/service/location.html diff --git a/onadata/libs/utils/csv_builder.py b/onadata/libs/utils/csv_builder.py index a2023d7597..6ea7293454 100644 --- a/onadata/libs/utils/csv_builder.py +++ b/onadata/libs/utils/csv_builder.py @@ -1,118 +1,163 @@ -from itertools import chain +# -*- coding=utf-8 -*- +""" +CSV export utility functions. +""" from collections import OrderedDict +from itertools import chain -import unicodecsv as csv from django.conf import settings from django.db.models.query import QuerySet -from django.utils.translation import ugettext as _ -from future.utils import iteritems -from past.builtins import basestring +from django.utils.translation import gettext as _ + +from six import iteritems + +import unicodecsv as csv from pyxform.question import Question from pyxform.section import RepeatingSection, Section from onadata.apps.logger.models import OsmData from onadata.apps.logger.models.xform import XForm, question_types_to_exclude from onadata.apps.viewer.models.data_dictionary import DataDictionary -from onadata.apps.viewer.models.parsed_instance import (ParsedInstance, - query_data) +from onadata.apps.viewer.models.parsed_instance import ParsedInstance, query_data from onadata.libs.exceptions import NoRecordsFoundError -from onadata.libs.utils.export_tools import str_to_bool from onadata.libs.utils.common_tags import ( - ATTACHMENTS, BAMBOO_DATASET_ID, DATE_MODIFIED, DELETEDAT, DURATION, - EDITED, GEOLOCATION, ID, MEDIA_ALL_RECEIVED, MEDIA_COUNT, NA_REP, - NOTES, STATUS, SUBMISSION_TIME, SUBMITTED_BY, TAGS, TOTAL_MEDIA, - UUID, VERSION, XFORM_ID_STRING, REVIEW_STATUS, REVIEW_COMMENT, - MULTIPLE_SELECT_TYPE, SELECT_BIND_TYPE, REVIEW_DATE) -from onadata.libs.utils.export_builder import (get_choice_label, - get_value_or_attachment_uri, - track_task_progress) + ATTACHMENTS, + BAMBOO_DATASET_ID, + DATE_MODIFIED, + DELETEDAT, + DURATION, + EDITED, + GEOLOCATION, + ID, + MEDIA_ALL_RECEIVED, + MEDIA_COUNT, + MULTIPLE_SELECT_TYPE, + NA_REP, + NOTES, + REVIEW_COMMENT, + REVIEW_DATE, + REVIEW_STATUS, + SELECT_BIND_TYPE, + STATUS, + SUBMISSION_TIME, + SUBMITTED_BY, + TAGS, + TOTAL_MEDIA, + UUID, + VERSION, + XFORM_ID_STRING, +) +from onadata.libs.utils.export_builder import ( + get_choice_label, + get_value_or_attachment_uri, + track_task_progress, +) +from onadata.libs.utils.export_tools import str_to_bool from onadata.libs.utils.model_tools import get_columns_with_hxl # the bind type of select multiples that we use to compare -MULTIPLE_SELECT_BIND_TYPE = u"select" -GEOPOINT_BIND_TYPE = u"geopoint" +MULTIPLE_SELECT_BIND_TYPE = "select" +GEOPOINT_BIND_TYPE = "geopoint" # column group delimiters -GROUP_DELIMITER_SLASH = '/' -GROUP_DELIMITER_DOT = '.' +GROUP_DELIMITER_SLASH = "/" +GROUP_DELIMITER_DOT = "." DEFAULT_GROUP_DELIMITER = GROUP_DELIMITER_SLASH GROUP_DELIMITERS = [GROUP_DELIMITER_SLASH, GROUP_DELIMITER_DOT] -DEFAULT_NA_REP = getattr(settings, 'NA_REP', NA_REP) +DEFAULT_NA_REP = getattr(settings, "NA_REP", NA_REP) # index tags -DEFAULT_OPEN_TAG = '[' -DEFAULT_CLOSE_TAG = ']' +DEFAULT_OPEN_TAG = "[" +DEFAULT_CLOSE_TAG = "]" DEFAULT_INDEX_TAGS = (DEFAULT_OPEN_TAG, DEFAULT_CLOSE_TAG) YES = 1 NO = 0 +# pylint: disable=invalid-name def remove_dups_from_list_maintain_order(lst): + """Removes duplicates from a list and still maintains the order.""" return list(OrderedDict.fromkeys(lst)) def get_prefix_from_xpath(xpath): + """Returns xpath prefix.""" xpath = str(xpath) - parts = xpath.rsplit('/', 1) + parts = xpath.rsplit("/", 1) if len(parts) == 1: return None - elif len(parts) == 2: - return '%s/' % parts[0] - else: - raise ValueError( - '%s cannot be prefixed, it returns %s' % (xpath, str(parts))) + if len(parts) == 2: + return f"{parts[0]}/" + raise ValueError(f"{xpath} cannot be prefixed, it returns {str(parts)}") -def get_labels_from_columns(columns, dd, group_delimiter, language=None): +def get_labels_from_columns(columns, data_dictionary, group_delimiter, language=None): + """Return ``column`` labels""" labels = [] for col in columns: - elem = dd.get_survey_element(col) - label = dd.get_label(col, elem=elem, - language=language) if elem else col - if elem is not None and elem.type == '': + elem = data_dictionary.get_survey_element(col) + label = ( + data_dictionary.get_label(col, elem=elem, language=language) + if elem + else col + ) + if elem is not None and elem.type == "": label = group_delimiter.join([elem.parent.name, label]) - if label == '': + if label == "": label = elem.name labels.append(label) return labels -def get_column_names_only(columns, dd, group_delimiter): +# pylint: disable=unused-argument +def get_column_names_only(columns, data_dictionary, group_delimiter): + """Return column names as a list.""" new_columns = [] for col in columns: new_col = None - elem = dd.get_survey_element(col) + elem = data_dictionary.get_survey_element(col) if elem is None: new_col = col - elif elem.type != '': + elif elem.type != "": new_col = elem.name else: - new_col = DEFAULT_GROUP_DELIMITER.join([ - elem.parent.name, - elem.name - ]) + new_col = DEFAULT_GROUP_DELIMITER.join([elem.parent.name, elem.name]) new_columns.append(new_col) return new_columns -def write_to_csv(path, rows, columns, columns_with_hxl=None, - remove_group_name=False, dd=None, - group_delimiter=DEFAULT_GROUP_DELIMITER, include_labels=False, - include_labels_only=False, include_hxl=False, - win_excel_utf8=False, total_records=None, - index_tags=DEFAULT_INDEX_TAGS, language=None): - na_rep = getattr(settings, 'NA_REP', NA_REP) - encoding = 'utf-8-sig' if win_excel_utf8 else 'utf-8' - with open(path, 'wb') as csvfile: - writer = csv.writer(csvfile, encoding=encoding, lineterminator='\n') +# pylint: disable=unused-argument,too-many-arguments,too-many-locals +def write_to_csv( + path, + rows, + columns, + columns_with_hxl=None, + remove_group_name=False, + data_dictionary=None, + group_delimiter=DEFAULT_GROUP_DELIMITER, + include_labels=False, + include_labels_only=False, + include_hxl=False, + win_excel_utf8=False, + total_records=None, + index_tags=DEFAULT_INDEX_TAGS, + language=None, +): + """Writes ``rows`` to a file in CSV format.""" + na_rep = getattr(settings, "NA_REP", NA_REP) + encoding = "utf-8-sig" if win_excel_utf8 else "utf-8" + with open(path, "wb") as csvfile: + writer = csv.writer(csvfile, encoding=encoding, lineterminator="\n") # Check if to truncate the group name prefix if not include_labels_only: - if remove_group_name and dd: - new_cols = get_column_names_only(columns, dd, group_delimiter) + if remove_group_name and data_dictionary: + new_cols = get_column_names_only( + columns, data_dictionary, group_delimiter + ) else: new_cols = columns @@ -126,13 +171,15 @@ def write_to_csv(path, rows, columns, columns_with_hxl=None, writer.writerow(new_cols) if include_labels or include_labels_only: - labels = get_labels_from_columns(columns, dd, group_delimiter, - language=language) + labels = get_labels_from_columns( + columns, data_dictionary, group_delimiter, language=language + ) writer.writerow(labels) if include_hxl and columns_with_hxl: - hxl_row = [columns_with_hxl.get(col, '') for col in columns] - hxl_row and writer.writerow(hxl_row) + hxl_row = [columns_with_hxl.get(col, "") for col in columns] + if hxl_row: + writer.writerow(hxl_row) for i, row in enumerate(rows, start=1): for col in AbstractDataFrameBuilder.IGNORED_COLUMNS: @@ -141,65 +188,112 @@ def write_to_csv(path, rows, columns, columns_with_hxl=None, track_task_progress(i, total_records) -class AbstractDataFrameBuilder(object): - IGNORED_COLUMNS = [XFORM_ID_STRING, STATUS, ATTACHMENTS, GEOLOCATION, - BAMBOO_DATASET_ID, DELETEDAT, EDITED] +# pylint: disable=too-few-public-methods,too-many-instance-attributes +class AbstractDataFrameBuilder: + """ + Abstract Data Frame Builder class + """ + + IGNORED_COLUMNS = [ + XFORM_ID_STRING, + STATUS, + ATTACHMENTS, + GEOLOCATION, + BAMBOO_DATASET_ID, + DELETEDAT, + EDITED, + ] # fields NOT within the form def that we want to include ADDITIONAL_COLUMNS = [ - ID, UUID, SUBMISSION_TIME, DATE_MODIFIED, TAGS, NOTES, VERSION, - DURATION, SUBMITTED_BY, TOTAL_MEDIA, MEDIA_COUNT, - MEDIA_ALL_RECEIVED] + ID, + UUID, + SUBMISSION_TIME, + DATE_MODIFIED, + TAGS, + NOTES, + VERSION, + DURATION, + SUBMITTED_BY, + TOTAL_MEDIA, + MEDIA_COUNT, + MEDIA_ALL_RECEIVED, + ] BINARY_SELECT_MULTIPLES = False VALUE_SELECT_MULTIPLES = False """ Group functionality used by any DataFrameBuilder i.e. XLS, CSV and KML """ - def __init__(self, username, id_string, filter_query=None, - group_delimiter=DEFAULT_GROUP_DELIMITER, - split_select_multiples=True, binary_select_multiples=False, - start=None, end=None, remove_group_name=False, xform=None, - include_labels=False, include_labels_only=False, - include_images=True, include_hxl=False, - win_excel_utf8=False, total_records=None, - index_tags=DEFAULT_INDEX_TAGS, value_select_multiples=False, - show_choice_labels=True, include_reviews=False, - language=None): + # pylint: disable=too-many-arguments,too-many-locals + def __init__( + self, + username, + id_string, + filter_query=None, + group_delimiter=DEFAULT_GROUP_DELIMITER, + split_select_multiples=True, + binary_select_multiples=False, + start=None, + end=None, + remove_group_name=False, + xform=None, + include_labels=False, + include_labels_only=False, + include_images=True, + include_hxl=False, + win_excel_utf8=False, + total_records=None, + index_tags=DEFAULT_INDEX_TAGS, + value_select_multiples=False, + show_choice_labels=True, + include_reviews=False, + language=None, + ): self.username = username self.id_string = id_string self.filter_query = filter_query self.group_delimiter = group_delimiter self.split_select_multiples = split_select_multiples + # pylint: disable=invalid-name self.BINARY_SELECT_MULTIPLES = binary_select_multiples self.VALUE_SELECT_MULTIPLES = value_select_multiples self.start = start self.end = end self.remove_group_name = remove_group_name - self.extra_columns = ( - self.ADDITIONAL_COLUMNS + getattr(settings, 'EXTRA_COLUMNS', [])) + self.extra_columns = self.ADDITIONAL_COLUMNS + getattr( + settings, "EXTRA_COLUMNS", [] + ) if include_reviews: self.extra_columns = self.extra_columns + [ - REVIEW_STATUS, REVIEW_COMMENT, REVIEW_DATE] + REVIEW_STATUS, + REVIEW_COMMENT, + REVIEW_DATE, + ] if xform: self.xform = xform else: - self.xform = XForm.objects.get(id_string=self.id_string, - user__username=self.username) + self.xform = XForm.objects.get( + id_string=self.id_string, user__username=self.username + ) self.include_labels = include_labels self.include_labels_only = include_labels_only self.include_images = include_images self.include_hxl = include_hxl self.win_excel_utf8 = win_excel_utf8 self.total_records = total_records - if index_tags != DEFAULT_INDEX_TAGS and \ - not isinstance(index_tags, (tuple, list)): - raise ValueError(_( - "Invalid option for repeat_index_tags: %s " - "expecting a tuple with opening and closing tags " - "e.g repeat_index_tags=('[', ']')" % index_tags)) + if index_tags != DEFAULT_INDEX_TAGS and not isinstance( + index_tags, (tuple, list) + ): + raise ValueError( + _( + f"Invalid option for repeat_index_tags: {index_tags} " + "expecting a tuple with opening and closing tags " + "e.g repeat_index_tags=('[', ']')" + ) + ) self.index_tags = index_tags self.show_choice_labels = show_choice_labels self.language = language @@ -207,47 +301,74 @@ def __init__(self, username, id_string, filter_query=None, self._setup() def _setup(self): - self.dd = self.xform - self.select_multiples = self._collect_select_multiples(self.dd, - self.language) - self.gps_fields = self._collect_gps_fields(self.dd) + self.data_dictionary = self.xform + self.select_multiples = self._collect_select_multiples( + self.data_dictionary, self.language + ) + self.gps_fields = self._collect_gps_fields(self.data_dictionary) @classmethod - def _fields_to_select(cls, dd): - return [c.get_abbreviated_xpath() - for c in dd.get_survey_elements() if isinstance(c, Question)] + def _fields_to_select(cls, data_dictionary): + return [ + c.get_abbreviated_xpath() + for c in data_dictionary.get_survey_elements() + if isinstance(c, Question) + ] @classmethod - def _collect_select_multiples(cls, dd, language=None): + def _collect_select_multiples(cls, data_dictionary, language=None): select_multiples = [] select_multiple_elements = [ - e for e in dd.get_survey_elements_with_choices() - if e.bind.get('type') == SELECT_BIND_TYPE - and e.type == MULTIPLE_SELECT_TYPE + e + for e in data_dictionary.get_survey_elements_with_choices() + if e.bind.get("type") == SELECT_BIND_TYPE and e.type == MULTIPLE_SELECT_TYPE ] for e in select_multiple_elements: xpath = e.get_abbreviated_xpath() - choices = [(c.get_abbreviated_xpath(), c.name, - get_choice_label(c.label, dd, language)) - for c in e.children] + choices = [ + ( + c.get_abbreviated_xpath(), + c.name, + get_choice_label(c.label, data_dictionary, language), + ) + for c in e.children + ] is_choice_randomized = str_to_bool( - e.parameters and e.parameters.get('randomize')) - if ((not choices and e.choice_filter) or is_choice_randomized) \ - and e.itemset: - itemset = dd.survey.to_json_dict()['choices'].get(e.itemset) - choices = [(u'/'.join([xpath, i.get('name')]), i.get('name'), - get_choice_label(i.get('label'), dd, language)) - for i in itemset] if itemset else choices + e.parameters and e.parameters.get("randomize") + ) + if ( + (not choices and e.choice_filter) or is_choice_randomized + ) and e.itemset: + itemset = data_dictionary.survey.to_json_dict()["choices"].get( + e.itemset + ) + choices = ( + [ + ( + "/".join([xpath, i.get("name")]), + i.get("name"), + get_choice_label(i.get("label"), data_dictionary, language), + ) + for i in itemset + ] + if itemset + else choices + ) select_multiples.append((xpath, choices)) return dict(select_multiples) + # pylint: disable=too-many-arguments @classmethod - def _split_select_multiples(cls, record, select_multiples, - binary_select_multiples=False, - value_select_multiples=False, - show_choice_labels=False): - """ Prefix contains the xpath and slash if we are within a repeat so + def _split_select_multiples( + cls, + record, + select_multiples, + binary_select_multiples=False, + value_select_multiples=False, + show_choice_labels=False, + ): + """Prefix contains the xpath and slash if we are within a repeat so that we can figure out which select multiples belong to which repeat """ for key, choices in select_multiples.items(): @@ -257,31 +378,44 @@ def _split_select_multiples(cls, record, select_multiples, if key in record: # split selected choices by spaces and join by / to the # element's xpath - selections = ["%s/%s" % (key, r) - for r in record[key].split(" ")] + selections = [f"{key}/{r}" for r in record[key].split(" ")] if value_select_multiples: - record.update(dict([ - (choice.replace('/' + name, '/' + label) - if show_choice_labels else choice, - (label if show_choice_labels else - record[key].split()[selections.index(choice)]) - if choice in selections else None) - for choice, name, label in choices])) + record.update( + { + choice.replace("/" + name, "/" + label) + if show_choice_labels + else choice: ( + label + if show_choice_labels + else record[key].split()[selections.index(choice)] + ) + if choice in selections + else None + for choice, name, label in choices + } + ) elif not binary_select_multiples: # add columns to record for every choice, with default # False and set to True for items in selections - record.update(dict([ - (choice.replace('/' + name, '/' + label) - if show_choice_labels else choice, - choice in selections) - for choice, name, label in choices])) + record.update( + { + choice.replace("/" + name, "/" + label) + if show_choice_labels + else choice: choice in selections + for choice, name, label in choices + } + ) else: record.update( - dict([ - (choice.replace('/' + name, '/' + label) - if show_choice_labels else choice, - YES if choice in selections else NO) - for choice, name, label in choices])) + { + choice.replace("/" + name, "/" + label) + if show_choice_labels + else choice: YES + if choice in selections + else NO + for choice, name, label in choices + } + ) # remove the column since we are adding separate columns # for each choice record.pop(key) @@ -292,40 +426,45 @@ def _split_select_multiples(cls, record, select_multiples, for list_item in record_item: if isinstance(list_item, dict): cls._split_select_multiples( - list_item, select_multiples, + list_item, + select_multiples, binary_select_multiples=binary_select_multiples, # noqa value_select_multiples=value_select_multiples, - show_choice_labels=show_choice_labels) + show_choice_labels=show_choice_labels, + ) return record @classmethod - def _collect_gps_fields(cls, dd): - return [e.get_abbreviated_xpath() for e in dd.get_survey_elements() - if e.bind.get("type") == "geopoint"] + def _collect_gps_fields(cls, data_dictionary): + return [ + e.get_abbreviated_xpath() + for e in data_dictionary.get_survey_elements() + if e.bind.get("type") == "geopoint" + ] @classmethod def _tag_edit_string(cls, record): """ Turns a list of tags into a string representation. """ - if '_tags' in record: + if "_tags" in record: tags = [] - for tag in record['_tags']: - if ',' in tag and ' ' in tag: - tags.append('"%s"' % tag) + for tag in record["_tags"]: + if "," in tag and " " in tag: + tags.append(f'"{tag}"') else: tags.append(tag) - record.update({'_tags': u', '.join(sorted(tags))}) + record.update({"_tags": ", ".join(sorted(tags))}) @classmethod def _split_gps_fields(cls, record, gps_fields): updated_gps_fields = {} for (key, value) in iteritems(record): - if key in gps_fields and isinstance(value, basestring): + if key in gps_fields and isinstance(value, str): gps_xpaths = DataDictionary.get_additional_geopoint_xpaths(key) - gps_parts = dict([(xpath, None) for xpath in gps_xpaths]) + gps_parts = {xpath: None for xpath in gps_xpaths} # hack, check if its a list and grab the object within that - parts = value.split(' ') + parts = value.split(" ") # TODO: check whether or not we can have a gps recording # from ODKCollect that has less than four components, # for now we are assuming that this is not the case. @@ -339,19 +478,25 @@ def _split_gps_fields(cls, record, gps_fields): cls._split_gps_fields(list_item, gps_fields) record.update(updated_gps_fields) - def _query_data(self, query='{}', start=0, - limit=ParsedInstance.DEFAULT_LIMIT, - fields='[]', count=False): + # pylint: disable=too-many-arguments + def _query_data( + self, + query="{}", + start=0, + limit=ParsedInstance.DEFAULT_LIMIT, + fields="[]", + count=False, + ): # query_data takes params as json strings # so we dumps the fields dictionary count_args = { - 'xform': self.xform, - 'query': query, - 'start': self.start, - 'end': self.end, - 'fields': '[]', - 'sort': '{}', - 'count': True + "xform": self.xform, + "query": query, + "start": self.start, + "end": self.end, + "fields": "[]", + "sort": "{}", + "count": True, } count_object = list(query_data(**count_args)) record_count = count_object[0]["count"] @@ -360,83 +505,128 @@ def _query_data(self, query='{}', start=0, # if count was requested, return the count if count: return record_count - else: - query_args = { - 'xform': self.xform, - 'query': query, - 'fields': fields, - 'start': self.start, - 'end': self.end, - # TODO: we might want to add this in for the user - # to sepcify a sort order - 'sort': 'id', - 'start_index': start, - 'limit': limit, - 'count': False - } - cursor = query_data(**query_args) - - return cursor + query_args = { + "xform": self.xform, + "query": query, + "fields": fields, + "start": self.start, + "end": self.end, + # TODO: we might want to add this in for the user + # to sepcify a sort order + "sort": "id", + "start_index": start, + "limit": limit, + "count": False, + } + cursor = query_data(**query_args) -class CSVDataFrameBuilder(AbstractDataFrameBuilder): + return cursor - def __init__(self, username, id_string, filter_query=None, - group_delimiter=DEFAULT_GROUP_DELIMITER, - split_select_multiples=True, binary_select_multiples=False, - start=None, end=None, remove_group_name=False, xform=None, - include_labels=False, include_labels_only=False, - include_images=False, include_hxl=False, - win_excel_utf8=False, total_records=None, - index_tags=DEFAULT_INDEX_TAGS, value_select_multiples=False, - show_choice_labels=False, include_reviews=False, - language=None): - - super(CSVDataFrameBuilder, self).__init__( - username, id_string, filter_query, group_delimiter, - split_select_multiples, binary_select_multiples, start, end, - remove_group_name, xform, include_labels, include_labels_only, - include_images, include_hxl, win_excel_utf8, total_records, - index_tags, value_select_multiples, - show_choice_labels, include_reviews, language) - self.ordered_columns = OrderedDict() - self.image_xpaths = [] if not self.include_images \ - else self.dd.get_media_survey_xpaths() +# pylint: disable=too-few-public-methods +class CSVDataFrameBuilder(AbstractDataFrameBuilder): + """CSV data frame builder""" + + # pylint: disable=too-many-arguments,too-many-locals + def __init__( + self, + username, + id_string, + filter_query=None, + group_delimiter=DEFAULT_GROUP_DELIMITER, + split_select_multiples=True, + binary_select_multiples=False, + start=None, + end=None, + remove_group_name=False, + xform=None, + include_labels=False, + include_labels_only=False, + include_images=False, + include_hxl=False, + win_excel_utf8=False, + total_records=None, + index_tags=DEFAULT_INDEX_TAGS, + value_select_multiples=False, + show_choice_labels=False, + include_reviews=False, + language=None, + ): + + super().__init__( + username, + id_string, + filter_query, + group_delimiter, + split_select_multiples, + binary_select_multiples, + start, + end, + remove_group_name, + xform, + include_labels, + include_labels_only, + include_images, + include_hxl, + win_excel_utf8, + total_records, + index_tags, + value_select_multiples, + show_choice_labels, + include_reviews, + language, + ) - def _setup(self): - super(CSVDataFrameBuilder, self)._setup() + self.ordered_columns = OrderedDict() + self.image_xpaths = ( + [] + if not self.include_images + else self.data_dictionary.get_media_survey_xpaths() + ) + # pylint: disable=too-many-arguments,too-many-branches,too-many-locals @classmethod - def _reindex(cls, key, value, ordered_columns, row, data_dictionary, - parent_prefix=None, - include_images=True, split_select_multiples=True, - index_tags=DEFAULT_INDEX_TAGS, show_choice_labels=False, - language=None): + def _reindex( + cls, + key, + value, + ordered_columns, + row, + data_dictionary, + parent_prefix=None, + include_images=True, + split_select_multiples=True, + index_tags=DEFAULT_INDEX_TAGS, + show_choice_labels=False, + language=None, + ): """ Flatten list columns by appending an index, otherwise return as is """ + def get_ordered_repeat_value(xpath, repeat_value): """ Return OrderedDict of repeats in the order in which they appear in the XForm. """ - children = data_dictionary.get_child_elements( - xpath, split_select_multiples) + children = data_dictionary.get_child_elements(xpath, split_select_multiples) item = OrderedDict() for elem in children: if not question_types_to_exclude(elem.type): - xp = elem.get_abbreviated_xpath() - item[xp] = repeat_value.get(xp, DEFAULT_NA_REP) + abbreviated_xpath = elem.get_abbreviated_xpath() + item[abbreviated_xpath] = repeat_value.get( + abbreviated_xpath, DEFAULT_NA_REP + ) return item - d = {} + record = {} # check for lists - if isinstance(value, list) and len(value) > 0 \ - and key not in [ATTACHMENTS, NOTES]: + # pylint: disable=too-many-nested-blocks + if isinstance(value, list) and value and key not in [ATTACHMENTS, NOTES]: for index, item in enumerate(value): # start at 1 index += 1 @@ -452,69 +642,85 @@ def get_ordered_repeat_value(xpath, repeat_value): # "children/details/immunization/polio_1", # generate ["children", index, "immunization/polio_1"] if parent_prefix is not None: - _key = '/'.join( - parent_prefix + - key.split('/')[len(parent_prefix):]) - xpaths = ['{key}{open_tag}{index}{close_tag}' - .format(key=_key, - open_tag=index_tags[0], - index=index, - close_tag=index_tags[1])] + \ - nested_key.split('/')[len(_key.split('/')):] + _key = "/".join( + parent_prefix + key.split("/")[len(parent_prefix) :] + ) + xpaths = [ + f"{_key}{index_tags[0]}{index}{index_tags[1]}" + ] + nested_key.split("/")[len(_key.split("/")) :] else: - xpaths = ['{key}{open_tag}{index}{close_tag}' - .format(key=key, - open_tag=index_tags[0], - index=index, - close_tag=index_tags[1])] + \ - nested_key.split('/')[len(key.split('/')):] + xpaths = [ + f"{key}{index_tags[0]}{index}{index_tags[1]}" + ] + nested_key.split("/")[len(key.split("/")) :] # re-create xpath the split on / xpaths = "/".join(xpaths).split("/") new_prefix = xpaths[:-1] if isinstance(nested_val, list): # if nested_value is a list, rinse and repeat - d.update(cls._reindex( - nested_key, nested_val, - ordered_columns, row, data_dictionary, - new_prefix, - include_images=include_images, - split_select_multiples=split_select_multiples, - index_tags=index_tags, - show_choice_labels=show_choice_labels, - language=language)) + record.update( + cls._reindex( + nested_key, + nested_val, + ordered_columns, + row, + data_dictionary, + new_prefix, + include_images=include_images, + split_select_multiples=split_select_multiples, + index_tags=index_tags, + show_choice_labels=show_choice_labels, + language=language, + ) + ) else: # it can only be a scalar # collapse xpath - new_xpath = u"/".join(xpaths) + new_xpath = "/".join(xpaths) # check if this key exists in our ordered columns if key in list(ordered_columns): if new_xpath not in ordered_columns[key]: ordered_columns[key].append(new_xpath) - d[new_xpath] = get_value_or_attachment_uri( - nested_key, nested_val, row, data_dictionary, + record[new_xpath] = get_value_or_attachment_uri( + nested_key, + nested_val, + row, + data_dictionary, include_images, show_choice_labels=show_choice_labels, - language=language) + language=language, + ) else: - d[key] = get_value_or_attachment_uri( - key, value, row, data_dictionary, include_images, + record[key] = get_value_or_attachment_uri( + key, + value, + row, + data_dictionary, + include_images, show_choice_labels=show_choice_labels, - language=language) + language=language, + ) else: # anything that's not a list will be in the top level dict so its # safe to simply assign if key == NOTES: # Do not include notes - d[key] = u"" + record[key] = "" else: - d[key] = get_value_or_attachment_uri( - key, value, row, data_dictionary, include_images, - show_choice_labels=show_choice_labels, language=language) - return d + record[key] = get_value_or_attachment_uri( + key, + value, + row, + data_dictionary, + include_images, + show_choice_labels=show_choice_labels, + language=language, + ) + return record @classmethod - def _build_ordered_columns(cls, survey_element, ordered_columns, - is_repeating_section=False): + def _build_ordered_columns( + cls, survey_element, ordered_columns, is_repeating_section=False + ): """ Build a flat ordered dict of column groups @@ -522,17 +728,17 @@ def _build_ordered_columns(cls, survey_element, ordered_columns, are not considered columns """ for child in survey_element.children: - # child_xpath = child.get_abbreviated_xpath() if isinstance(child, Section): child_is_repeating = False if isinstance(child, RepeatingSection): ordered_columns[child.get_abbreviated_xpath()] = [] child_is_repeating = True - cls._build_ordered_columns(child, ordered_columns, - child_is_repeating) - elif isinstance(child, Question) and not \ - question_types_to_exclude(child.type) and not\ - is_repeating_section: # if is_repeating_section, + cls._build_ordered_columns(child, ordered_columns, child_is_repeating) + elif ( + isinstance(child, Question) + and not question_types_to_exclude(child.type) + and not is_repeating_section + ): # if is_repeating_section, # its parent already initiliased an empty list # so we dont add it to our list of columns, # the repeating columns list will be @@ -550,15 +756,18 @@ def _update_ordered_columns_from_data(self, cursor): for (key, choices) in iteritems(self.select_multiples): # HACK to ensure choices are NOT duplicated if key in self.ordered_columns.keys(): - self.ordered_columns[key] = \ - remove_dups_from_list_maintain_order( - [choice.replace('/' + name, '/' + label) - if self.show_choice_labels else choice - for choice, name, label in choices]) + self.ordered_columns[key] = remove_dups_from_list_maintain_order( + [ + choice.replace("/" + name, "/" + label) + if self.show_choice_labels + else choice + for choice, name, label in choices + ] + ) # add ordered columns for gps fields for key in self.gps_fields: - gps_xpaths = self.dd.get_additional_geopoint_xpaths(key) + gps_xpaths = self.data_dictionary.get_additional_geopoint_xpaths(key) self.ordered_columns[key] = [key] + gps_xpaths # add ordered columns for nested repeat data @@ -566,12 +775,17 @@ def _update_ordered_columns_from_data(self, cursor): # re index column repeats for (key, value) in iteritems(record): self._reindex( - key, value, self.ordered_columns, record, self.dd, + key, + value, + self.ordered_columns, + record, + self.data_dictionary, include_images=self.image_xpaths, split_select_multiples=self.split_select_multiples, index_tags=self.index_tags, show_choice_labels=self.show_choice_labels, - language=self.language) + language=self.language, + ) def _format_for_dataframe(self, cursor): """ @@ -581,10 +795,12 @@ def _format_for_dataframe(self, cursor): # split select multiples if self.split_select_multiples: record = self._split_select_multiples( - record, self.select_multiples, + record, + self.select_multiples, self.BINARY_SELECT_MULTIPLES, self.VALUE_SELECT_MULTIPLES, - show_choice_labels=self.show_choice_labels) + show_choice_labels=self.show_choice_labels, + ) # check for gps and split into # components i.e. latitude, longitude, # altitude, precision @@ -594,22 +810,29 @@ def _format_for_dataframe(self, cursor): # re index repeats for (key, value) in iteritems(record): reindexed = self._reindex( - key, value, self.ordered_columns, record, self.dd, + key, + value, + self.ordered_columns, + record, + self.data_dictionary, include_images=self.image_xpaths, split_select_multiples=self.split_select_multiples, index_tags=self.index_tags, show_choice_labels=self.show_choice_labels, - language=self.language) + language=self.language, + ) flat_dict.update(reindexed) yield flat_dict def export_to(self, path, dataview=None): + """Export a CSV formated to the given ``path``.""" self.ordered_columns = OrderedDict() - self._build_ordered_columns(self.dd.survey, self.ordered_columns) + self._build_ordered_columns(self.data_dictionary.survey, self.ordered_columns) if dataview: - cursor = dataview.query_data(dataview, all_data=True, - filter_query=self.filter_query) + cursor = dataview.query_data( + dataview, all_data=True, filter_query=self.filter_query + ) if isinstance(cursor, QuerySet): cursor = cursor.iterator() @@ -617,11 +840,15 @@ def export_to(self, path, dataview=None): data = self._format_for_dataframe(cursor) - columns = list(chain.from_iterable( - [[xpath] if cols is None else cols - for (xpath, cols) in iteritems(self.ordered_columns) - if [c for c in dataview.columns if xpath.startswith(c)]] - )) + columns = list( + chain.from_iterable( + [ + [xpath] if cols is None else cols + for (xpath, cols) in iteritems(self.ordered_columns) + if [c for c in dataview.columns if xpath.startswith(c)] + ] + ) + ) else: try: cursor = self._query_data(self.filter_query) @@ -637,29 +864,40 @@ def export_to(self, path, dataview=None): # Unpack xform columns and data data = self._format_for_dataframe(cursor) - columns = list(chain.from_iterable( - [[xpath] if cols is None else cols - for (xpath, cols) in iteritems(self.ordered_columns)])) + columns = list( + chain.from_iterable( + [ + [xpath] if cols is None else cols + for (xpath, cols) in iteritems(self.ordered_columns) + ] + ) + ) # add extra columns - columns += [col for col in self.extra_columns] + columns += list(self.extra_columns) - for field in self.dd.get_survey_elements_of_type('osm'): - columns += OsmData.get_tag_keys(self.xform, - field.get_abbreviated_xpath(), - include_prefix=True) + for field in self.data_dictionary.get_survey_elements_of_type("osm"): + columns += OsmData.get_tag_keys( + self.xform, field.get_abbreviated_xpath(), include_prefix=True + ) columns_with_hxl = self.include_hxl and get_columns_with_hxl( - self.dd.survey_elements) - - write_to_csv(path, data, columns, - columns_with_hxl=columns_with_hxl, - remove_group_name=self.remove_group_name, - dd=self.dd, group_delimiter=self.group_delimiter, - include_labels=self.include_labels, - include_labels_only=self.include_labels_only, - include_hxl=self.include_hxl, - win_excel_utf8=self.win_excel_utf8, - total_records=self.total_records, - index_tags=self.index_tags, - language=self.language) + self.data_dictionary.survey_elements + ) + + write_to_csv( + path, + data, + columns, + columns_with_hxl=columns_with_hxl, + remove_group_name=self.remove_group_name, + data_dictionary=self.data_dictionary, + group_delimiter=self.group_delimiter, + include_labels=self.include_labels, + include_labels_only=self.include_labels_only, + include_hxl=self.include_hxl, + win_excel_utf8=self.win_excel_utf8, + total_records=self.total_records, + index_tags=self.index_tags, + language=self.language, + ) diff --git a/onadata/libs/utils/csv_import.py b/onadata/libs/utils/csv_import.py index f791d6fa6e..948ddb4aa3 100644 --- a/onadata/libs/utils/csv_import.py +++ b/onadata/libs/utils/csv_import.py @@ -3,7 +3,6 @@ CSV data import module. """ import functools -import json import logging import sys import uuid @@ -24,8 +23,8 @@ from django.contrib.auth.models import User from django.core.files.storage import default_storage from django.utils import timezone -from django.utils.translation import ugettext as _ -from future.utils import iteritems +from django.utils.translation import gettext as _ +from six import iteritems from multidb.pinning import use_master from onadata.apps.logger.models import Instance, XForm @@ -101,11 +100,12 @@ def dict2xmlsubmission(submission_dict, xform, instance_id, submission_date): :rtype: string """ + xform_dict = xform.json_dict() return ( '' '<{0} id="{1}" instanceID="uuid:{2}" submissionDate="{3}">{4}' "".format( - json.loads(xform.json).get("name", xform.id_string), + xform_dict.get("name", xform.id_string), xform.id_string, instance_id, submission_date, @@ -323,7 +323,7 @@ def submit_csv(username, xform, csv_file, overwrite=False): csv_file.seek(0) csv_reader = ucsv.DictReader(csv_file, encoding="utf-8-sig") - xform_json = json.loads(xform.json) + xform_json = xform.json_dict() select_multiples = [ qstn.name for qstn in xform.get_survey_elements_of_type(MULTIPLE_SELECT_TYPE) ] diff --git a/onadata/libs/utils/decorators.py b/onadata/libs/utils/decorators.py index 89e1d19fa4..faa39d7814 100644 --- a/onadata/libs/utils/decorators.py +++ b/onadata/libs/utils/decorators.py @@ -1,13 +1,17 @@ +# -*- coding: utf-8 -*- +"""decorators module""" from functools import wraps -from future.moves.urllib.parse import urlparse +from six.moves.urllib.parse import urlparse from django.contrib.auth import REDIRECT_FIELD_NAME -from django.utils.decorators import available_attrs +from django.contrib.auth.views import redirect_to_login from django.conf import settings from django.http import HttpResponseRedirect def check_obj(f): + """Checks if the first argument is truthy and then calls the underlying function.""" + # pylint: disable=inconsistent-return-statements @wraps(f) def with_check_obj(*args, **kwargs): if args[0]: @@ -17,24 +21,27 @@ def with_check_obj(*args, **kwargs): def is_owner(view_func): - @wraps(view_func, assigned=available_attrs(view_func)) + """Redirects to login if not owner.""" + + @wraps(view_func) def _wrapped_view(request, *args, **kwargs): # assume username is first arg if request.user.is_authenticated: - if request.user.username == kwargs['username']: + if request.user.username == kwargs["username"]: return view_func(request, *args, **kwargs) protocol = "https" if request.is_secure() else "http" - return HttpResponseRedirect("%s://%s" % (protocol, - request.get_host())) + return HttpResponseRedirect(f"{protocol}://{request.get_host()}") path = request.build_absolute_uri() login_url = request.build_absolute_uri(settings.LOGIN_URL) # If the login url is the same scheme and net location then just # use the path as the "next" url. login_scheme, login_netloc = urlparse(login_url)[:2] current_scheme, current_netloc = urlparse(path)[:2] - if ((not login_scheme or login_scheme == current_scheme) and - (not login_netloc or login_netloc == current_netloc)): + is_scheme = not login_scheme or login_scheme == current_scheme + is_netloc = not login_netloc or login_netloc == current_netloc + if is_scheme and is_netloc: path = request.get_full_path() - from django.contrib.auth.views import redirect_to_login + return redirect_to_login(path, None, REDIRECT_FIELD_NAME) + return _wrapped_view diff --git a/onadata/libs/utils/dict_tools.py b/onadata/libs/utils/dict_tools.py index f57c87c450..a348bf26aa 100644 --- a/onadata/libs/utils/dict_tools.py +++ b/onadata/libs/utils/dict_tools.py @@ -4,8 +4,6 @@ """ import json -from past.builtins import basestring - def get_values_matching_key(doc, key): """ @@ -37,7 +35,7 @@ def list_to_dict(items, value): key = items.pop() result = {} - bracket_index = key.find('[') + bracket_index = key.find("[") if bracket_index > 0: value = [value] @@ -55,18 +53,21 @@ def merge_list_of_dicts(list_of_dicts, override_keys: list = None): """ result = {} + # pylint: disable=too-many-nested-blocks for row in list_of_dicts: for k, v in row.items(): if isinstance(v, list): - z = merge_list_of_dicts(result[k] + v if k in result else v, - override_keys=override_keys) + z = merge_list_of_dicts( + result[k] + v if k in result else v, override_keys=override_keys + ) result[k] = z if isinstance(z, list) else [z] else: if k in result: if isinstance(v, dict): try: result[k] = merge_list_of_dicts( - [result[k], v], override_keys=override_keys) + [result[k], v], override_keys=override_keys + ) except AttributeError as e: # If the key is within the override_keys # (Is a select_multiple question) We make @@ -74,12 +75,15 @@ def merge_list_of_dicts(list_of_dicts, override_keys: list = None): # more accurate as they usually mean that # the select_multiple has been split into # separate columns for each choice - if override_keys and isinstance(result[k], str)\ - and k in override_keys: + if ( + override_keys + and isinstance(result[k], str) + and k in override_keys + ): result[k] = {} result[k] = merge_list_of_dicts( - [result[k], v], - override_keys=override_keys) + [result[k], v], override_keys=override_keys + ) else: raise e else: @@ -95,11 +99,11 @@ def remove_indices_from_dict(obj): Removes indices from a obj dict. """ if not isinstance(obj, dict): - raise ValueError(u"Expecting a dict, found: {}".format(type(obj))) + raise ValueError(f"Expecting a dict, found: {type(obj)}") result = {} for key, val in obj.items(): - bracket_index = key.find('[') + bracket_index = key.find("[") key = key[:bracket_index] if bracket_index > -1 else key val = remove_indices_from_dict(val) if isinstance(val, dict) else val if isinstance(val, list): @@ -126,7 +130,7 @@ def csv_dict_to_nested_dict(csv_dict, select_multiples=None): for key in list(csv_dict): result = {} value = csv_dict[key] - split_keys = key.split('/') + split_keys = key.split("/") if len(split_keys) == 1: result[key] = value @@ -147,8 +151,8 @@ def dict_lists2strings(adict): :param d: The dict to convert. :returns: The converted dict.""" for k, v in adict.items(): - if isinstance(v, list) and all([isinstance(e, basestring) for e in v]): - adict[k] = ' '.join(v) + if isinstance(v, list) and all(isinstance(e, str) for e in v): + adict[k] = " ".join(v) elif isinstance(v, dict): adict[k] = dict_lists2strings(v) @@ -162,8 +166,8 @@ def dict_paths2dict(adict): result = {} for k, v in adict.items(): - if k.find('/') > 0: - parts = k.split('/') + if k.find("/") > 0: + parts = k.split("/") if len(parts) > 1: k = parts[0] for part in parts[1:]: @@ -179,9 +183,9 @@ def query_list_to_dict(query_list_str): Returns a 'label' and 'text' from a Rapidpro values JSON string as a dict. """ data_list = json.loads(query_list_str) - data_dict = dict() + data_dict = {} for value in data_list: - data_dict[value['label']] = value['text'] + data_dict[value["label"]] = value["text"] return data_dict @@ -190,7 +194,7 @@ def floip_response_headers_dict(data, xform_headers): """ Returns a dict from matching xform headers and floip responses. """ - headers = [i.split('/')[-1] for i in xform_headers] + headers = [i.split("/")[-1] for i in xform_headers] data = [i[4] for i in data] flow_dict = dict(zip(headers, data)) diff --git a/onadata/libs/utils/email.py b/onadata/libs/utils/email.py index d67a9d4731..d725941b6f 100644 --- a/onadata/libs/utils/email.py +++ b/onadata/libs/utils/email.py @@ -1,84 +1,77 @@ +# -*- coding: utf-8 -*- +""" +email utility functions. +""" from django.conf import settings from django.core.mail import EmailMultiAlternatives from django.template.loader import render_to_string -from future.moves.urllib.parse import urlencode +from six.moves.urllib.parse import urlencode from rest_framework.reverse import reverse def get_verification_url(redirect_url, request, verification_key): + """Returns the verification_url""" verification_url = getattr(settings, "VERIFICATION_URL", None) - url = verification_url or reverse( - 'userprofile-verify-email', request=request - ) - query_params_dict = {'verification_key': verification_key} - redirect_url and query_params_dict.update({ - 'redirect_url': redirect_url - }) + url = verification_url or reverse("userprofile-verify-email", request=request) + query_params_dict = {"verification_key": verification_key} + if redirect_url: + query_params_dict.update({"redirect_url": redirect_url}) query_params_string = urlencode(query_params_dict) - verification_url = '{}?{}'.format(url, query_params_string) + verification_url = f"{url}?{query_params_string}" return verification_url def get_verification_email_data(email, username, verification_url, request): - email_data = {'email': email} + """ + Returns the verification email content + """ + email_data = {"email": email} ctx_dict = { - 'username': username, - 'expiration_days': getattr(settings, "ACCOUNT_ACTIVATION_DAYS", 1), - 'verification_url': verification_url + "username": username, + "expiration_days": getattr(settings, "ACCOUNT_ACTIVATION_DAYS", 1), + "verification_url": verification_url, } key_template_path_dict = { - 'subject': 'registration/verification_email_subject.txt', - 'message_txt': 'registration/verification_email.txt' + "subject": "registration/verification_email_subject.txt", + "message_txt": "registration/verification_email.txt", } for key, template_path in key_template_path_dict.items(): - email_data.update({ - key: render_to_string( - template_path, - ctx_dict, - request=request - ) - }) + email_data.update( + {key: render_to_string(template_path, ctx_dict, request=request)} + ) return email_data -def get_account_lockout_email_data(username, ip, end=False): +def get_account_lockout_email_data(username, ip_address, end=False): """Generates both the email upon start and end of account lockout""" - message_path = 'account_lockout/lockout_start.txt' - subject_path = 'account_lockout/lockout_email_subject.txt' + message_path = "account_lockout/lockout_start.txt" + subject_path = "account_lockout/lockout_email_subject.txt" if end: - message_path = 'account_lockout/lockout_end.txt' + message_path = "account_lockout/lockout_end.txt" ctx_dict = { - 'username': username, - 'remote_ip': ip, - 'lockout_time': getattr(settings, 'LOCKOUT_TIME', 1800) / 60, - 'support_email': getattr( - settings, 'SUPPORT_EMAIL', 'support@example.com') + "username": username, + "remote_ip": ip_address, + "lockout_time": getattr(settings, "LOCKOUT_TIME", 1800) / 60, + "support_email": getattr(settings, "SUPPORT_EMAIL", "support@example.com"), } email_data = { - 'subject': render_to_string(subject_path), - 'message_txt': render_to_string(message_path, ctx_dict) + "subject": render_to_string(subject_path), + "message_txt": render_to_string(message_path, ctx_dict), } return email_data def send_generic_email(email, message_txt, subject): - if any(a in [None, ''] for a in [email, message_txt, subject]): - raise ValueError( - "email, message_txt amd subject arguments are ALL required." - ) + if any(a in [None, ""] for a in [email, message_txt, subject]): + raise ValueError("email, message_txt amd subject arguments are ALL required.") from_email = settings.DEFAULT_FROM_EMAIL - email_message = EmailMultiAlternatives( - subject, - message_txt, - from_email, - [email] - ) + email_message = EmailMultiAlternatives(subject, message_txt, from_email, [email]) email_message.send() diff --git a/onadata/libs/utils/export_builder.py b/onadata/libs/utils/export_builder.py index 9d0249737c..f2538d5a9a 100644 --- a/onadata/libs/utils/export_builder.py +++ b/onadata/libs/utils/export_builder.py @@ -1,3 +1,4 @@ +# pylint: disable=too-many-lines # -*- coding: utf-8 -*- """ ExportBuilder @@ -5,49 +6,74 @@ from __future__ import unicode_literals import csv -import logging -import sys -import uuid import re -from builtins import str as text -from datetime import datetime, date -from zipfile import ZipFile, ZIP_DEFLATED +import uuid +from datetime import date, datetime +from zipfile import ZIP_DEFLATED, ZipFile -from celery import current_task from django.conf import settings +from django.contrib.sites.models import Site from django.core.files.temp import NamedTemporaryFile -from onadata.libs.utils.common_tools import str_to_bool -from django.utils.translation import ugettext as _ -from future.utils import iteritems + +from six import iteritems + +from celery import current_task from openpyxl.utils.datetime import to_excel from openpyxl.workbook import Workbook from pyxform.question import Question from pyxform.section import RepeatingSection, Section -from savReaderWriter import SavWriter + + +try: + from savReaderWriter import SavWriter +except ImportError: + SavWriter = None from onadata.apps.logger.models.osmdata import OsmData -from onadata.apps.logger.models.xform import (QUESTION_TYPES_TO_EXCLUDE, - _encode_for_mongo) +from onadata.apps.logger.models.xform import ( + QUESTION_TYPES_TO_EXCLUDE, + _encode_for_mongo, +) from onadata.apps.viewer.models.data_dictionary import DataDictionary from onadata.libs.utils.common_tags import ( - ATTACHMENTS, BAMBOO_DATASET_ID, DELETEDAT, DURATION, GEOLOCATION, - ID, INDEX, MULTIPLE_SELECT_TYPE, SELECT_ONE, NOTES, PARENT_INDEX, - PARENT_TABLE_NAME, REPEAT_INDEX_TAGS, SAV_255_BYTES_TYPE, - SAV_NUMERIC_TYPE, STATUS, SUBMISSION_TIME, SUBMITTED_BY, TAGS, UUID, - VERSION, XFORM_ID_STRING, REVIEW_STATUS, REVIEW_COMMENT, SELECT_BIND_TYPE, - REVIEW_DATE) + ATTACHMENTS, + BAMBOO_DATASET_ID, + DELETEDAT, + DURATION, + GEOLOCATION, + ID, + INDEX, + MULTIPLE_SELECT_TYPE, + NOTES, + PARENT_INDEX, + PARENT_TABLE_NAME, + REPEAT_INDEX_TAGS, + REVIEW_COMMENT, + REVIEW_DATE, + REVIEW_STATUS, + SAV_255_BYTES_TYPE, + SAV_NUMERIC_TYPE, + SELECT_BIND_TYPE, + SELECT_ONE, + STATUS, + SUBMISSION_TIME, + SUBMITTED_BY, + TAGS, + UUID, + VERSION, + XFORM_ID_STRING, +) +from onadata.libs.utils.common_tools import str_to_bool from onadata.libs.utils.mongo import _decode_from_mongo, _is_invalid_for_mongo + # the bind type of select multiples that we use to compare -GEOPOINT_BIND_TYPE = 'geopoint' -OSM_BIND_TYPE = 'osm' +GEOPOINT_BIND_TYPE = "geopoint" +OSM_BIND_TYPE = "osm" DEFAULT_UPDATE_BATCH = 100 YES = 1 NO = 0 -# savReaderWriter behaves differenlty depending on this -IS_PY_3K = sys.version_info[0] > 2 - def current_site_url(path): """ @@ -55,15 +81,15 @@ def current_site_url(path): :param path :return: complete url """ - from django.contrib.sites.models import Site + current_site = Site.objects.get_current() - protocol = getattr(settings, 'ONA_SITE_PROTOCOL', 'http') - port = getattr(settings, 'ONA_SITE_PORT', '') - url = '%s://%s' % (protocol, current_site.domain) + protocol = getattr(settings, "ONA_SITE_PROTOCOL", "http") + port = getattr(settings, "ONA_SITE_PORT", "") + url = f"{protocol}://{current_site.domain}" if port: - url += ':%s' % port + url += f":{port}" if path: - url += '%s' % path + url += f"{path}" return url @@ -73,9 +99,12 @@ def get_choice_label(label, data_dictionary, language=None): Return the label matching selected language or simply just the label. """ if isinstance(label, dict): - languages = [i for i in label.keys()] - _language = language if language in languages else \ - data_dictionary.get_language(languages) + languages = list(label.keys()) + _language = ( + language + if language in languages + else data_dictionary.get_language(languages) + ) return label[_language] @@ -87,12 +116,12 @@ def get_choice_label_value(key, value, data_dictionary, language=None): Return the label of a choice matching the value if the key xpath is a SELECT_ONE otherwise it returns the value unchanged. """ + def _get_choice_label_value(lookup): _label = None for choice in data_dictionary.get_survey_element(key).children: if choice.name == lookup: - _label = get_choice_label(choice.label, data_dictionary, - language) + _label = get_choice_label(choice.label, data_dictionary, language) break return _label @@ -103,27 +132,35 @@ def _get_choice_label_value(lookup): if key in data_dictionary.get_select_multiple_xpaths(): answers = [] - for item in value.split(' '): + for item in value.split(" "): answer = _get_choice_label_value(item) answers.append(answer or item) if [_i for _i in answers if _i is not None]: - label = ' '.join(answers) + label = " ".join(answers) return label or value -def get_value_or_attachment_uri( # pylint: disable=too-many-arguments - key, value, row, data_dictionary, media_xpaths, - attachment_list=None, show_choice_labels=False, language=None): +# pylint: disable=too-many-arguments +def get_value_or_attachment_uri( + key, + value, + row, + data_dictionary, + media_xpaths, + attachment_list=None, + show_choice_labels=False, + language=None, +): """ - Gets either the attachment value or the attachment url - :param key: used to retrieve survey element - :param value: filename - :param row: current records row - :param data_dictionary: form structure - :param include_images: boolean value to either inlcude images or not - :param attachment_list: to be used incase row doesn't have ATTACHMENTS key - :return: value + Gets either the attachment value or the attachment url + :param key: used to retrieve survey element + :param value: filename + :param row: current records row + :param data_dictionary: form structure + :param include_images: boolean value to either inlcude images or not + :param attachment_list: to be used incase row doesn't have ATTACHMENTS key + :return: value """ if show_choice_labels: value = get_choice_label_value(key, value, data_dictionary, language) @@ -135,45 +172,49 @@ def get_value_or_attachment_uri( # pylint: disable=too-many-arguments attachments = [ a for a in row.get(ATTACHMENTS, attachment_list or []) - if a.get('name') == value + if a.get("name") == value ] if attachments: - value = current_site_url(attachments[0].get('download_url', '')) + value = current_site_url(attachments[0].get("download_url", "")) return value def get_data_dictionary_from_survey(survey): - dd = DataDictionary() - dd._survey = survey + """Creates a DataDictionary instance from an XML survey instance.""" + data_dicionary = DataDictionary() + # pylint: disable=protected-access + data_dicionary._survey = survey - return dd + return data_dicionary def encode_if_str(row, key, encode_dates=False, sav_writer=None): + """Encode a string value in ``row[key]``.""" val = row.get(key) if isinstance(val, (datetime, date)): if sav_writer: if isinstance(val, datetime): - if len(val.isoformat()): - strptime_fmt = '%Y-%m-%dT%H:%M:%S' + if val.isoformat(): + strptime_fmt = "%Y-%m-%dT%H:%M:%S" else: - strptime_fmt = '%Y-%m-%dT%H:%M:%S.%f%z' + strptime_fmt = "%Y-%m-%dT%H:%M:%S.%f%z" else: - strptime_fmt = '%Y-%m-%d' - return sav_writer.spssDateTime(val.isoformat().encode('utf-8'), - strptime_fmt) - elif encode_dates: + strptime_fmt = "%Y-%m-%d" + return sav_writer.spssDateTime( + val.isoformat().encode("utf-8"), strptime_fmt + ) + if encode_dates: return val.isoformat() if sav_writer: - val = '' if val is None else val - return text(val) if IS_PY_3K and not isinstance(val, bool) else val + val = "" if val is None else val + return str(val) if not isinstance(val, bool) else val return val -def dict_to_joined_export(data, index, indices, name, survey, row, - media_xpaths=[]): +# pylint: disable=too-many-arguments,too-many-locals,too-many-branches +def dict_to_joined_export(data, index, indices, name, survey, row, media_xpaths=None): """ Converts a dict into one or more tabular datasets :param data: current record which can be changed or updated @@ -184,7 +225,8 @@ def dict_to_joined_export(data, index, indices, name, survey, row, :param row: current record that remains unchanged on this function's recall """ output = {} - # TODO: test for _geolocation and attachment lists + media_xpaths = [] if media_xpaths is None else media_xpaths + # pylint: disable=too-many-nested-blocks if isinstance(data, dict): for (key, val) in iteritems(data): if isinstance(val, list) and key not in [NOTES, ATTACHMENTS, TAGS]: @@ -195,10 +237,13 @@ def dict_to_joined_export(data, index, indices, name, survey, row, indices[key] += 1 child_index = indices[key] new_output = dict_to_joined_export( - child, child_index, indices, key, survey, row, - media_xpaths) - d = {INDEX: child_index, PARENT_INDEX: index, - PARENT_TABLE_NAME: name} + child, child_index, indices, key, survey, row, media_xpaths + ) + item = { + INDEX: child_index, + PARENT_INDEX: index, + PARENT_TABLE_NAME: name, + } # iterate over keys within new_output and append to # main output for (out_key, out_val) in iteritems(new_output): @@ -207,22 +252,26 @@ def dict_to_joined_export(data, index, indices, name, survey, row, output[out_key] = [] output[out_key].extend(out_val) else: - d.update(out_val) - output[key].append(d) + item.update(out_val) + output[key].append(item) else: if name not in output: output[name] = {} if key in [TAGS]: - output[name][key] = ','.join(val) + output[name][key] = ",".join(val) elif key in [NOTES]: - note_list = [v if isinstance(v, text) - else v['note'] for v in val] - output[name][key] = '\r\n'.join(note_list) + note_list = [v if isinstance(v, str) else v["note"] for v in val] + output[name][key] = "\r\n".join(note_list) else: data_dictionary = get_data_dictionary_from_survey(survey) output[name][key] = get_value_or_attachment_uri( - key, val, data, data_dictionary, media_xpaths, - row and row.get(ATTACHMENTS)) + key, + val, + data, + data_dictionary, + media_xpaths, + row and row.get(ATTACHMENTS), + ) return output @@ -239,16 +288,20 @@ def is_all_numeric(items): for i in items: float(i) # if there is a zero padded number, it is not all numeric - if isinstance(i, text) and len(i) > 1 and \ - i[0] == '0' and i[1] != '.': + if isinstance(i, str) and len(i) > 1 and i[0] == "0" and i[1] != ".": return False return True except ValueError: return False # check for zero padded numbers to be treated as non numeric - return not (any([i.startswith('0') and len(i) > 1 and i.find('.') == -1 - for i in items if isinstance(i, text)])) + return not ( + any( + i.startswith("0") and len(i) > 1 and i.find(".") == -1 + for i in items + if isinstance(i, str) + ) + ) def track_task_progress(additions, total=None): @@ -260,29 +313,28 @@ def track_task_progress(additions, total=None): :param total: :return: """ - try: - if additions % getattr(settings, 'EXPORT_TASK_PROGRESS_UPDATE_BATCH', - DEFAULT_UPDATE_BATCH) == 0: - meta = {'progress': additions} - if total: - meta.update({'total': total}) - current_task.update_state(state='PROGRESS', meta=meta) - except Exception as e: - logging.exception( - _('Track task progress threw exception: %s' % text(e))) + batch_size = getattr( + settings, "EXPORT_TASK_PROGRESS_UPDATE_BATCH", DEFAULT_UPDATE_BATCH + ) + if additions % batch_size == 0: + meta = {"progress": additions} + if total: + meta.update({"total": total}) + current_task.update_state(state="PROGRESS", meta=meta) +# pylint: disable=invalid-name def string_to_date_with_xls_validation(date_str): - """ Try to convert a string to a date object. + """Try to convert a string to a date object. :param date_str: string to convert :returns: object if converted, otherwise date string """ - if not isinstance(date_str, text): + if not isinstance(date_str, str): return date_str try: - date_obj = datetime.strptime(date_str, '%Y-%m-%d').date() + date_obj = datetime.strptime(date_str, "%Y-%m-%d").date() to_excel(date_obj) except ValueError: return date_str @@ -290,8 +342,9 @@ def string_to_date_with_xls_validation(date_str): return date_obj +# pylint: disable=invalid-name def decode_mongo_encoded_section_names(data): - """ Recursively decode mongo keys. + """Recursively decode mongo keys. :param data: A dictionary to decode. """ @@ -301,32 +354,92 @@ def decode_mongo_encoded_section_names(data): if isinstance(v, dict): new_v = decode_mongo_encoded_section_names(v) elif isinstance(v, list): - new_v = [decode_mongo_encoded_section_names(x) - if isinstance(x, dict) else x for x in v] + new_v = [ + decode_mongo_encoded_section_names(x) if isinstance(x, dict) else x + for x in v + ] results[_decode_from_mongo(k)] = new_v return results -class ExportBuilder(object): - IGNORED_COLUMNS = [XFORM_ID_STRING, STATUS, ATTACHMENTS, GEOLOCATION, - BAMBOO_DATASET_ID, DELETEDAT] +def _check_sav_column(column, columns): + """ + Check for duplicates and append @ 4 chars uuid. + Also checks for column length more than 64 chars + :param column: + :return: truncated column + """ + + if len(column) > 64: + col_len_diff = len(column) - 64 + column = column[:-col_len_diff] + + if column.lower() in (t.lower() for t in columns): + if len(column) > 59: + column = column[:-5] + column = column + "@" + str(uuid.uuid4()).split("-")[1] + + return column + + +def _get_var_name(title, var_names): + """ + GET valid SPSS varName. + @param title - survey element title/name + @param var_names - list of existing var_names + @return valid varName and list of var_names with new var name appended + """ + var_name = ( + title.replace("/", ".") + .replace("-", "_") + .replace(":", "_") + .replace("{", "") + .replace("}", "") + ) + var_name = _check_sav_column(var_name, var_names) + var_name = "@" + var_name if var_name.startswith("_") else var_name + var_names.append(var_name) + return var_name, var_names + + +# pylint: disable=too-many-instance-attributes +class ExportBuilder: + """Utility class for generating multiple formats of data export to file.""" + + IGNORED_COLUMNS = [ + XFORM_ID_STRING, + STATUS, + ATTACHMENTS, + GEOLOCATION, + BAMBOO_DATASET_ID, + DELETEDAT, + ] # fields we export but are not within the form's structure EXTRA_FIELDS = [ - ID, UUID, SUBMISSION_TIME, INDEX, PARENT_TABLE_NAME, PARENT_INDEX, - TAGS, NOTES, VERSION, DURATION, - SUBMITTED_BY] + ID, + UUID, + SUBMISSION_TIME, + INDEX, + PARENT_TABLE_NAME, + PARENT_INDEX, + TAGS, + NOTES, + VERSION, + DURATION, + SUBMITTED_BY, + ] SPLIT_SELECT_MULTIPLES = True BINARY_SELECT_MULTIPLES = False VALUE_SELECT_MULTIPLES = False # column group delimiters get_value_or_attachment_uri - GROUP_DELIMITER_SLASH = '/' - GROUP_DELIMITER_DOT = '.' + GROUP_DELIMITER_SLASH = "/" + GROUP_DELIMITER_DOT = "." GROUP_DELIMITER = GROUP_DELIMITER_SLASH GROUP_DELIMITERS = [GROUP_DELIMITER_SLASH, GROUP_DELIMITER_DOT] # index tags - REPEAT_INDEX_TAGS = ('[', ']') + REPEAT_INDEX_TAGS = ("[", "]") INCLUDE_LABELS = False INCLUDE_LABELS_ONLY = False @@ -336,12 +449,12 @@ class ExportBuilder(object): SHOW_CHOICE_LABELS = False INCLUDE_REVIEWS = False - TYPES_TO_CONVERT = ['int', 'decimal', 'date'] # , 'dateTime'] + TYPES_TO_CONVERT = ["int", "decimal", "date"] # , 'dateTime'] CONVERT_FUNCS = { - 'int': int, - 'decimal': float, - 'date': string_to_date_with_xls_validation, - 'dateTime': lambda x: datetime.strptime(x[:19], '%Y-%m-%dT%H:%M:%S') + "int": int, + "decimal": float, + "date": string_to_date_with_xls_validation, + "dateTime": lambda x: datetime.strptime(x[:19], "%Y-%m-%dT%H:%M:%S"), } TRUNCATE_GROUP_TITLE = False @@ -351,13 +464,18 @@ class ExportBuilder(object): language = None def __init__(self): - self.extra_columns = ( - self.EXTRA_FIELDS + getattr(settings, 'EXTRA_COLUMNS', [])) + self.extra_columns = self.EXTRA_FIELDS + getattr(settings, "EXTRA_COLUMNS", []) self.osm_columns = [] @classmethod - def format_field_title(cls, abbreviated_xpath, field_delimiter, - data_dictionary, remove_group_name=False): + def format_field_title( + cls, + abbreviated_xpath, + field_delimiter, + data_dictionary, + remove_group_name=False, + ): + """Format the field title.""" title = abbreviated_xpath # Check if to truncate the group name prefix if remove_group_name: @@ -365,187 +483,265 @@ def format_field_title(cls, abbreviated_xpath, field_delimiter, # incase abbreviated_xpath is a choices xpath if elem is None: pass - elif elem.type == '': - title = '/'.join([elem.parent.name, elem.name]) + elif elem.type == "": + title = "/".join([elem.parent.name, elem.name]) else: title = elem.name - if field_delimiter != '/': - title = field_delimiter.join(title.split('/')) + if field_delimiter != "/": + title = field_delimiter.join(title.split("/")) return title def get_choice_label_from_dict(self, label): + """Returns the choice label for the default language.""" if isinstance(label, dict): language = self.get_default_language(list(label)) label = label.get(self.language or language) return label - def _get_select_mulitples_choices(self, child, dd, field_delimiter, - remove_group_name): + def _get_select_mulitples_choices( + self, child, data_dicionary, field_delimiter, remove_group_name + ): def get_choice_dict(xpath, label): title = ExportBuilder.format_field_title( - xpath, field_delimiter, dd, remove_group_name + xpath, field_delimiter, data_dicionary, remove_group_name ) return { - 'label': field_delimiter.join([child.name, label or title]), - '_label': label or title, - '_label_xpath': field_delimiter.join([child.name, - label or title]), - 'title': title, - 'xpath': xpath, - 'type': 'string' + "label": field_delimiter.join([child.name, label or title]), + "_label": label or title, + "_label_xpath": field_delimiter.join([child.name, label or title]), + "title": title, + "xpath": xpath, + "type": "string", } choices = [] is_choice_randomized = str_to_bool( - child.parameters and child.parameters.get('randomize')) - if ((not child.children and child.choice_filter) - or is_choice_randomized) and child.itemset: - itemset = dd.survey.to_json_dict()['choices'].get(child.itemset) - choices = [get_choice_dict( - '/'.join([child.get_abbreviated_xpath(), i['name']]), - self.get_choice_label_from_dict(i['label']) - ) for i in itemset] if itemset else choices + child.parameters and child.parameters.get("randomize") + ) + if ( + (not child.children and child.choice_filter) or is_choice_randomized + ) and child.itemset: + itemset = data_dicionary.survey.to_json_dict()["choices"].get(child.itemset) + choices = ( + [ + get_choice_dict( + "/".join([child.get_abbreviated_xpath(), i["name"]]), + self.get_choice_label_from_dict(i["label"]), + ) + for i in itemset + ] + if itemset + else choices + ) else: - choices = [get_choice_dict( - c.get_abbreviated_xpath(), - get_choice_label(c.label, dd, language=self.language)) - for c in child.children] + choices = [ + get_choice_dict( + c.get_abbreviated_xpath(), + get_choice_label(c.label, data_dicionary, language=self.language), + ) + for c in child.children + ] return choices def set_survey(self, survey, xform=None, include_reviews=False): + """Set's the XForm XML ``survey`` instance.""" if self.INCLUDE_REVIEWS or include_reviews: + # pylint: disable=invalid-name self.EXTRA_FIELDS = self.EXTRA_FIELDS + [ - REVIEW_STATUS, REVIEW_COMMENT, REVIEW_DATE] + REVIEW_STATUS, + REVIEW_COMMENT, + REVIEW_DATE, + ] self.__init__() - dd = get_data_dictionary_from_survey(survey) + data_dicionary = get_data_dictionary_from_survey(survey) + # pylint: disable=too-many-locals,too-many-branches,too-many-arguments def build_sections( - current_section, survey_element, sections, select_multiples, - gps_fields, osm_fields, encoded_fields, select_ones, - field_delimiter='/', remove_group_name=False, language=None): + current_section, + survey_element, + sections, + select_multiples, + gps_fields, + osm_fields, + encoded_fields, + select_ones, + field_delimiter="/", + remove_group_name=False, + language=None, + ): + # pylint: disable=too-many-nested-blocks for child in survey_element.children: - current_section_name = current_section['name'] + current_section_name = current_section["name"] # if a section, recurs if isinstance(child, Section): # if its repeating, build a new section if isinstance(child, RepeatingSection): # section_name in recursive call changes section = { - 'name': child.get_abbreviated_xpath(), - 'elements': []} + "name": child.get_abbreviated_xpath(), + "elements": [], + } self.sections.append(section) build_sections( - section, child, sections, select_multiples, - gps_fields, osm_fields, encoded_fields, - select_ones, field_delimiter, remove_group_name, - language=language) + section, + child, + sections, + select_multiples, + gps_fields, + osm_fields, + encoded_fields, + select_ones, + field_delimiter, + remove_group_name, + language=language, + ) else: # its a group, recurs using the same section build_sections( - current_section, child, sections, select_multiples, - gps_fields, osm_fields, encoded_fields, - select_ones, field_delimiter, remove_group_name, - language=language) - elif isinstance(child, Question) and \ - (child.bind.get('type') - not in QUESTION_TYPES_TO_EXCLUDE and - child.type not in QUESTION_TYPES_TO_EXCLUDE): + current_section, + child, + sections, + select_multiples, + gps_fields, + osm_fields, + encoded_fields, + select_ones, + field_delimiter, + remove_group_name, + language=language, + ) + elif isinstance(child, Question) and ( + child.bind.get("type") not in QUESTION_TYPES_TO_EXCLUDE + and child.type not in QUESTION_TYPES_TO_EXCLUDE + ): # add to survey_sections if isinstance(child, Question): child_xpath = child.get_abbreviated_xpath() _title = ExportBuilder.format_field_title( child.get_abbreviated_xpath(), - field_delimiter, dd, remove_group_name + field_delimiter, + data_dicionary, + remove_group_name, + ) + _label = ( + data_dicionary.get_label( + child_xpath, elem=child, language=language + ) + or _title + ) + current_section["elements"].append( + { + "label": _label, + "title": _title, + "xpath": child_xpath, + "type": child.bind.get("type"), + } ) - _label = \ - dd.get_label( - child_xpath, - elem=child, - language=language) or _title - current_section['elements'].append({ - 'label': _label, - 'title': _title, - 'xpath': child_xpath, - 'type': child.bind.get('type') - }) if _is_invalid_for_mongo(child_xpath): if current_section_name not in encoded_fields: encoded_fields[current_section_name] = {} encoded_fields[current_section_name].update( - {child_xpath: _encode_for_mongo(child_xpath)}) + {child_xpath: _encode_for_mongo(child_xpath)} + ) # if its a select multiple, make columns out of its choices - if child.bind.get('type') == SELECT_BIND_TYPE \ - and child.type == MULTIPLE_SELECT_TYPE: + if ( + child.bind.get("type") == SELECT_BIND_TYPE + and child.type == MULTIPLE_SELECT_TYPE + ): choices = [] if self.SPLIT_SELECT_MULTIPLES: choices = self._get_select_mulitples_choices( - child, dd, field_delimiter, remove_group_name + child, + data_dicionary, + field_delimiter, + remove_group_name, ) for choice in choices: - if choice not in current_section['elements']: - current_section['elements'].append(choice) + if choice not in current_section["elements"]: + current_section["elements"].append(choice) # choices_xpaths = [c['xpath'] for c in choices] _append_xpaths_to_section( - current_section_name, select_multiples, - child.get_abbreviated_xpath(), choices) + current_section_name, + select_multiples, + child.get_abbreviated_xpath(), + choices, + ) # split gps fields within this section - if child.bind.get('type') == GEOPOINT_BIND_TYPE: + if child.bind.get("type") == GEOPOINT_BIND_TYPE: # add columns for geopoint components xpaths = DataDictionary.get_additional_geopoint_xpaths( - child.get_abbreviated_xpath()) + child.get_abbreviated_xpath() + ) for xpath in xpaths: _title = ExportBuilder.format_field_title( - xpath, field_delimiter, dd, - remove_group_name + xpath, + field_delimiter, + data_dicionary, + remove_group_name, + ) + current_section["elements"].append( + { + "label": _title, + "title": _title, + "xpath": xpath, + "type": "decimal", + } ) - current_section['elements'].append({ - 'label': _title, - 'title': _title, - 'xpath': xpath, - 'type': 'decimal' - }) _append_xpaths_to_section( - current_section_name, gps_fields, - child.get_abbreviated_xpath(), xpaths) + current_section_name, + gps_fields, + child.get_abbreviated_xpath(), + xpaths, + ) # get other osm fields - if child.get(u"type") == OSM_BIND_TYPE: + if child.get("type") == OSM_BIND_TYPE: xpaths = _get_osm_paths(child, xform) for xpath in xpaths: _title = ExportBuilder.format_field_title( - xpath, field_delimiter, dd, - remove_group_name + xpath, + field_delimiter, + data_dicionary, + remove_group_name, + ) + current_section["elements"].append( + { + "label": _title, + "title": _title, + "xpath": xpath, + "type": "osm", + } ) - current_section['elements'].append({ - 'label': _title, - 'title': _title, - 'xpath': xpath, - 'type': 'osm' - }) _append_xpaths_to_section( - current_section_name, osm_fields, - child.get_abbreviated_xpath(), xpaths) - if child.bind.get(u"type") == SELECT_BIND_TYPE \ - and child.type == SELECT_ONE: + current_section_name, + osm_fields, + child.get_abbreviated_xpath(), + xpaths, + ) + if ( + child.bind.get("type") == SELECT_BIND_TYPE + and child.type == SELECT_ONE + ): _append_xpaths_to_section( - current_section_name, select_ones, - child.get_abbreviated_xpath(), []) + current_section_name, + select_ones, + child.get_abbreviated_xpath(), + [], + ) - def _append_xpaths_to_section(current_section_name, field_list, xpath, - xpaths): + def _append_xpaths_to_section(current_section_name, field_list, xpath, xpaths): if current_section_name not in field_list: field_list[current_section_name] = {} - field_list[ - current_section_name][xpath] = xpaths + field_list[current_section_name][xpath] = xpaths def _get_osm_paths(osm_field, xform): """ @@ -555,38 +751,53 @@ def _get_osm_paths(osm_field, xform): osm_columns = [] if osm_field and xform: osm_columns = OsmData.get_tag_keys( - xform, osm_field.get_abbreviated_xpath(), - include_prefix=True) + xform, osm_field.get_abbreviated_xpath(), include_prefix=True + ) return osm_columns - self.dd = dd + # pylint: disable=attribute-defined-outside-init + self.data_dicionary = data_dicionary self.survey = survey self.select_multiples = {} self.select_ones = {} self.gps_fields = {} self.osm_fields = {} self.encoded_fields = {} - main_section = {'name': survey.name, 'elements': []} + main_section = {"name": survey.name, "elements": []} self.sections = [main_section] build_sections( - main_section, self.survey, self.sections, - self.select_multiples, self.gps_fields, self.osm_fields, - self.encoded_fields, self.select_ones, self.GROUP_DELIMITER, - self.TRUNCATE_GROUP_TITLE, language=self.language) + main_section, + self.survey, + self.sections, + self.select_multiples, + self.gps_fields, + self.osm_fields, + self.encoded_fields, + self.select_ones, + self.GROUP_DELIMITER, + self.TRUNCATE_GROUP_TITLE, + language=self.language, + ) def section_by_name(self, name): - matches = [s for s in self.sections if s['name'] == name] - assert(len(matches) == 1) + """Return section by the given ``name``.""" + matches = [s for s in self.sections if s["name"] == name] + assert len(matches) == 1 return matches[0] # pylint: disable=too-many-arguments @classmethod - def split_select_multiples(cls, row, select_multiples, - select_values=False, - binary_select_multiples=False, - show_choice_labels=False, data_dictionary=None, - language=None): + def split_select_multiples( + cls, + row, + select_multiples, + select_values=False, + binary_select_multiples=False, + show_choice_labels=False, + data_dictionary=None, + language=None, + ): """ Split select multiple choices in a submission to individual columns. @@ -609,41 +820,57 @@ def split_select_multiples(cls, row, select_multiples, # for each select_multiple, get the associated data and split it for (xpath, choices) in iteritems(select_multiples): # get the data matching this xpath - data = row.get(xpath) and text(row.get(xpath)) + data = row.get(xpath) and str(row.get(xpath)) selections = [] if data: - selections = [ - '{0}/{1}'.format( - xpath, selection) for selection in data.split()] + selections = [f"{xpath}/{selection}" for selection in data.split()] if show_choice_labels and data_dictionary: row[xpath] = get_choice_label_value( - xpath, data, data_dictionary, language) + xpath, data, data_dictionary, language + ) if select_values: if show_choice_labels: - row.update(dict( - [(choice['label'], choice['_label'] - if selections and choice['xpath'] in selections - else None) - for choice in choices])) + row.update( + { + choice["label"]: choice["_label"] + if selections and choice["xpath"] in selections + else None + for choice in choices + } + ) else: - row.update(dict( - [(choice['xpath'], - data.split()[selections.index(choice['xpath'])] - if selections and choice['xpath'] in selections - else None) - for choice in choices])) + row.update( + { + choice["xpath"]: data.split()[ + selections.index(choice["xpath"]) + ] + if selections and choice["xpath"] in selections + else None + for choice in choices + } + ) elif binary_select_multiples: - row.update(dict( - [(choice['label'] - if show_choice_labels else choice['xpath'], - YES if choice['xpath'] in selections else NO) - for choice in choices])) + row.update( + { + choice["label"] + if show_choice_labels + else choice["xpath"]: YES + if choice["xpath"] in selections + else NO + for choice in choices + } + ) else: - row.update(dict( - [(choice['label'] - if show_choice_labels else choice['xpath'], - choice['xpath'] in selections if selections else None) - for choice in choices])) + row.update( + { + choice["label"] + if show_choice_labels + else choice["xpath"]: choice["xpath"] in selections + if selections + else None + for choice in choices + } + ) return row @classmethod @@ -679,65 +906,75 @@ def convert_type(cls, value, data_type): except ValueError: return value + # pylint: disable=too-many-branches def pre_process_row(self, row, section): """ Split select multiples, gps and decode . and $ """ - section_name = section['name'] + section_name = section["name"] # first decode fields so that subsequent lookups # have decoded field names if section_name in self.encoded_fields: row = ExportBuilder.decode_mongo_encoded_fields( - row, self.encoded_fields[section_name]) + row, self.encoded_fields[section_name] + ) if section_name in self.select_multiples: select_multiples = self.select_multiples[section_name] if self.SPLIT_SELECT_MULTIPLES: row = ExportBuilder.split_select_multiples( - row, select_multiples, self.VALUE_SELECT_MULTIPLES, + row, + select_multiples, + self.VALUE_SELECT_MULTIPLES, self.BINARY_SELECT_MULTIPLES, show_choice_labels=self.SHOW_CHOICE_LABELS, - data_dictionary=self.dd, language=self.language) + data_dictionary=self.data_dicionary, + language=self.language, + ) if not self.SPLIT_SELECT_MULTIPLES and self.SHOW_CHOICE_LABELS: for xpath in select_multiples: # get the data matching this xpath - data = row.get(xpath) and text(row.get(xpath)) + data = row.get(xpath) and str(row.get(xpath)) if data: row[xpath] = get_choice_label_value( - xpath, data, self.dd, self.language) + xpath, data, self.data_dicionary, self.language + ) if section_name in self.gps_fields: - row = ExportBuilder.split_gps_components( - row, self.gps_fields[section_name]) + row = ExportBuilder.split_gps_components(row, self.gps_fields[section_name]) if section_name in self.select_ones and self.SHOW_CHOICE_LABELS: for key in self.select_ones[section_name]: if key in row: - row[key] = get_choice_label_value(key, row[key], self.dd, - self.language) + row[key] = get_choice_label_value( + key, row[key], self.data_dicionary, self.language + ) # convert to native types - for elm in section['elements']: + for elm in section["elements"]: # only convert if its in our list and its not empty, just to # optimize - value = row.get(elm['xpath']) - if elm['type'] in ExportBuilder.TYPES_TO_CONVERT\ - and value is not None and value != '': - row[elm['xpath']] = ExportBuilder.convert_type( - value, elm['type']) + value = row.get(elm["xpath"]) + if ( + elm["type"] in ExportBuilder.TYPES_TO_CONVERT + and value is not None + and value != "" + ): + row[elm["xpath"]] = ExportBuilder.convert_type(value, elm["type"]) if SUBMISSION_TIME in row: row[SUBMISSION_TIME] = ExportBuilder.convert_type( - row[SUBMISSION_TIME], 'dateTime') + row[SUBMISSION_TIME], "dateTime" + ) # Map dynamic values for key, value in row.items(): if isinstance(value, str): - dynamic_val_regex = '\$\{\w+\}' # noqa + dynamic_val_regex = r"\$\{\w+\}" # noqa # Find substrings that match ${`any_text`} result = re.findall(dynamic_val_regex, value) if result: for val in result: - val_key = val.replace('${', '').replace('}', '') + val_key = val.replace("${", "").replace("}", "") # Try retrieving value of ${`any_text`} from the # row data and replace the value if row.get(val_key): @@ -746,58 +983,64 @@ def pre_process_row(self, row, section): return row + # pylint: disable=too-many-locals,too-many-branches,unused-argument def to_zipped_csv(self, path, data, *args, **kwargs): + """Export CSV formatted files from ``data`` and zip the files.""" + def write_row(row, csv_writer, fields): - csv_writer.writerow( - [encode_if_str(row, field) for field in fields]) + csv_writer.writerow([encode_if_str(row, field) for field in fields]) csv_defs = {} - dataview = kwargs.get('dataview') - total_records = kwargs.get('total_records') + dataview = kwargs.get("dataview") + total_records = kwargs.get("total_records") for section in self.sections: - csv_file = NamedTemporaryFile(suffix='.csv', mode='w') + csv_file = NamedTemporaryFile(suffix=".csv", mode="w") csv_writer = csv.writer(csv_file) - csv_defs[section['name']] = { - 'csv_file': csv_file, 'csv_writer': csv_writer} + csv_defs[section["name"]] = {"csv_file": csv_file, "csv_writer": csv_writer} # write headers if not self.INCLUDE_LABELS_ONLY: for section in self.sections: - fields = self.get_fields(dataview, section, 'title') - csv_defs[section['name']]['csv_writer'].writerow( - [f for f in fields]) + fields = self.get_fields(dataview, section, "title") + csv_defs[section["name"]]["csv_writer"].writerow(list(fields)) # write labels if self.INCLUDE_LABELS or self.INCLUDE_LABELS_ONLY: for section in self.sections: - fields = self.get_fields(dataview, section, 'label') - csv_defs[section['name']]['csv_writer'].writerow( - [f for f in fields]) + fields = self.get_fields(dataview, section, "label") + csv_defs[section["name"]]["csv_writer"].writerow(list(fields)) - media_xpaths = [] if not self.INCLUDE_IMAGES \ - else self.dd.get_media_survey_xpaths() + media_xpaths = ( + [] + if not self.INCLUDE_IMAGES + else self.data_dicionary.get_media_survey_xpaths() + ) - columns_with_hxl = kwargs.get('columns_with_hxl') + columns_with_hxl = kwargs.get("columns_with_hxl") # write hxl row if self.INCLUDE_HXL and columns_with_hxl: for section in self.sections: - fields = self.get_fields(dataview, section, 'title') - hxl_row = [columns_with_hxl.get(col, '') - for col in fields] + fields = self.get_fields(dataview, section, "title") + hxl_row = [columns_with_hxl.get(col, "") for col in fields] if hxl_row: - writer = csv_defs[section['name']]['csv_writer'] + writer = csv_defs[section["name"]]["csv_writer"] writer.writerow(hxl_row) index = 1 indices = {} survey_name = self.survey.name - for i, d in enumerate(data, start=1): + for i, row_data in enumerate(data, start=1): # decode mongo section names - joined_export = dict_to_joined_export(d, index, indices, - survey_name, - self.survey, d, - media_xpaths) + joined_export = dict_to_joined_export( + row_data, + index, + indices, + survey_name, + self.survey, + row_data, + media_xpaths, + ) output = decode_mongo_encoded_section_names(joined_export) # attach meta fields (index, parent_index, parent_table) # output has keys for every section @@ -807,126 +1050,134 @@ def write_row(row, csv_writer, fields): output[survey_name][PARENT_INDEX] = -1 for section in self.sections: # get data for this section and write to csv - section_name = section['name'] + section_name = section["name"] csv_def = csv_defs[section_name] - fields = self.get_fields(dataview, section, 'xpath') - csv_writer = csv_def['csv_writer'] + fields = self.get_fields(dataview, section, "xpath") + csv_writer = csv_def["csv_writer"] # section name might not exist within the output, e.g. data was # not provided for said repeat - write test to check this row = output.get(section_name, None) if isinstance(row, dict): - write_row( - self.pre_process_row(row, section), - csv_writer, fields) + write_row(self.pre_process_row(row, section), csv_writer, fields) elif isinstance(row, list): for child_row in row: write_row( - self.pre_process_row(child_row, section), - csv_writer, fields) + self.pre_process_row(child_row, section), csv_writer, fields + ) index += 1 track_task_progress(i, total_records) # write zipfile - with ZipFile(path, 'w', ZIP_DEFLATED, allowZip64=True) as zip_file: + with ZipFile(path, "w", ZIP_DEFLATED, allowZip64=True) as zip_file: for (section_name, csv_def) in iteritems(csv_defs): - csv_file = csv_def['csv_file'] + csv_file = csv_def["csv_file"] csv_file.seek(0) zip_file.write( - csv_file.name, '_'.join(section_name.split('/')) + '.csv') + csv_file.name, "_".join(section_name.split("/")) + ".csv" + ) # close files when we are done for (section_name, csv_def) in iteritems(csv_defs): - csv_def['csv_file'].close() + csv_def["csv_file"].close() @classmethod def get_valid_sheet_name(cls, desired_name, existing_names): + """Returns a valid sheet_name based on the desired names""" # a sheet name has to be <= 31 characters and not a duplicate of an # existing sheet # truncate sheet_name to XLSDataFrameBuilder.SHEET_NAME_MAX_CHARS - new_sheet_name = \ - desired_name[:cls.XLS_SHEET_NAME_MAX_CHARS] + new_sheet_name = desired_name[: cls.XLS_SHEET_NAME_MAX_CHARS] # make sure its unique within the list i = 1 generated_name = new_sheet_name while generated_name in existing_names: - digit_length = len(text(i)) - allowed_name_len = cls.XLS_SHEET_NAME_MAX_CHARS - \ - digit_length + digit_length = len(str(i)) + allowed_name_len = cls.XLS_SHEET_NAME_MAX_CHARS - digit_length # make name the required len if len(generated_name) > allowed_name_len: generated_name = generated_name[:allowed_name_len] - generated_name = '{0}{1}'.format(generated_name, i) + generated_name = f"{generated_name}{i}" i += 1 return generated_name + # pylint: disable=too-many-locals,too-many-statements,unused-argument def to_xls_export(self, path, data, *args, **kwargs): + """Export data to a spreadsheet document.""" + def write_row(data, work_sheet, fields, work_sheet_titles): # update parent_table with the generated sheet's title - data[PARENT_TABLE_NAME] = work_sheet_titles.get( - data.get(PARENT_TABLE_NAME)) + data[PARENT_TABLE_NAME] = work_sheet_titles.get(data.get(PARENT_TABLE_NAME)) work_sheet.append([data.get(f) for f in fields]) - dataview = kwargs.get('dataview') - total_records = kwargs.get('total_records') + dataview = kwargs.get("dataview") + total_records = kwargs.get("total_records") - wb = Workbook(write_only=True) + work_book = Workbook(write_only=True) work_sheets = {} # map of section_names to generated_names work_sheet_titles = {} for section in self.sections: - section_name = section['name'] + section_name = section["name"] work_sheet_title = ExportBuilder.get_valid_sheet_name( - '_'.join(section_name.split('/')), work_sheet_titles.values()) + "_".join(section_name.split("/")), work_sheet_titles.values() + ) work_sheet_titles[section_name] = work_sheet_title - work_sheets[section_name] = wb.create_sheet( - title=work_sheet_title) + work_sheets[section_name] = work_book.create_sheet(title=work_sheet_title) # write the headers if not self.INCLUDE_LABELS_ONLY: for section in self.sections: - section_name = section['name'] - headers = self.get_fields(dataview, section, 'title') + section_name = section["name"] + headers = self.get_fields(dataview, section, "title") # get the worksheet - ws = work_sheets[section_name] - ws.append(headers) + work_sheet = work_sheets[section_name] + work_sheet.append(headers) # write labels if self.INCLUDE_LABELS or self.INCLUDE_LABELS_ONLY: for section in self.sections: - section_name = section['name'] - labels = self.get_fields(dataview, section, 'label') + section_name = section["name"] + labels = self.get_fields(dataview, section, "label") # get the worksheet - ws = work_sheets[section_name] - ws.append(labels) + work_sheet = work_sheets[section_name] + work_sheet.append(labels) - media_xpaths = [] if not self.INCLUDE_IMAGES \ - else self.dd.get_media_survey_xpaths() + media_xpaths = ( + [] + if not self.INCLUDE_IMAGES + else self.data_dicionary.get_media_survey_xpaths() + ) # write hxl header - columns_with_hxl = kwargs.get('columns_with_hxl') + columns_with_hxl = kwargs.get("columns_with_hxl") if self.INCLUDE_HXL and columns_with_hxl: for section in self.sections: - section_name = section['name'] - headers = self.get_fields(dataview, section, 'title') + section_name = section["name"] + headers = self.get_fields(dataview, section, "title") # get the worksheet - ws = work_sheets[section_name] + work_sheet = work_sheets[section_name] - hxl_row = [columns_with_hxl.get(col, '') - for col in headers] - hxl_row and ws.append(hxl_row) + hxl_row = [columns_with_hxl.get(col, "") for col in headers] + if hxl_row: + work_sheet.append(hxl_row) index = 1 indices = {} survey_name = self.survey.name - for i, d in enumerate(data, start=1): - joined_export = dict_to_joined_export(d, index, indices, - survey_name, - self.survey, d, - media_xpaths) + for i, row_data in enumerate(data, start=1): + joined_export = dict_to_joined_export( + row_data, + index, + indices, + survey_name, + self.survey, + row_data, + media_xpaths, + ) output = decode_mongo_encoded_section_names(joined_export) # attach meta fields (index, parent_index, parent_table) # output has keys for every section @@ -936,70 +1187,91 @@ def write_row(data, work_sheet, fields, work_sheet_titles): output[survey_name][PARENT_INDEX] = -1 for section in self.sections: # get data for this section and write to xls - section_name = section['name'] - fields = self.get_fields(dataview, section, 'xpath') + section_name = section["name"] + fields = self.get_fields(dataview, section, "xpath") - ws = work_sheets[section_name] + work_sheet = work_sheets[section_name] # section might not exist within the output, e.g. data was # not provided for said repeat - write test to check this row = output.get(section_name, None) if isinstance(row, dict): write_row( self.pre_process_row(row, section), - ws, fields, work_sheet_titles) + work_sheet, + fields, + work_sheet_titles, + ) elif isinstance(row, list): for child_row in row: write_row( self.pre_process_row(child_row, section), - ws, fields, work_sheet_titles) + work_sheet, + fields, + work_sheet_titles, + ) index += 1 track_task_progress(i, total_records) - wb.save(filename=path) + work_book.save(filename=path) - def to_flat_csv_export(self, path, data, username, id_string, - filter_query, **kwargs): + # pylint: disable=too-many-locals,unused-argument + def to_flat_csv_export( + self, path, data, username, id_string, filter_query, **kwargs + ): """ Generates a flattened CSV file for submitted data. """ - # TODO resolve circular import + # pylint: disable=import-outside-toplevel from onadata.libs.utils.csv_builder import CSVDataFrameBuilder - start = kwargs.get('start') - end = kwargs.get('end') - dataview = kwargs.get('dataview') - xform = kwargs.get('xform') - options = kwargs.get('options') - total_records = kwargs.get('total_records') - win_excel_utf8 = options.get('win_excel_utf8') if options else False + + start = kwargs.get("start") + end = kwargs.get("end") + dataview = kwargs.get("dataview") + xform = kwargs.get("xform") + options = kwargs.get("options") + total_records = kwargs.get("total_records") + win_excel_utf8 = options.get("win_excel_utf8") if options else False index_tags = options.get(REPEAT_INDEX_TAGS, self.REPEAT_INDEX_TAGS) - show_choice_labels = options.get('show_choice_labels', False) - language = options.get('language') + show_choice_labels = options.get("show_choice_labels", False) + language = options.get("language") csv_builder = CSVDataFrameBuilder( - username, id_string, filter_query, self.GROUP_DELIMITER, - self.SPLIT_SELECT_MULTIPLES, self.BINARY_SELECT_MULTIPLES, - start, end, self.TRUNCATE_GROUP_TITLE, xform, - self.INCLUDE_LABELS, self.INCLUDE_LABELS_ONLY, - self.INCLUDE_IMAGES, self.INCLUDE_HXL, - win_excel_utf8=win_excel_utf8, total_records=total_records, + username, + id_string, + filter_query, + self.GROUP_DELIMITER, + self.SPLIT_SELECT_MULTIPLES, + self.BINARY_SELECT_MULTIPLES, + start, + end, + self.TRUNCATE_GROUP_TITLE, + xform, + self.INCLUDE_LABELS, + self.INCLUDE_LABELS_ONLY, + self.INCLUDE_IMAGES, + self.INCLUDE_HXL, + win_excel_utf8=win_excel_utf8, + total_records=total_records, index_tags=index_tags, value_select_multiples=self.VALUE_SELECT_MULTIPLES, show_choice_labels=show_choice_labels, - include_reviews=self.INCLUDE_REVIEWS, language=language) + include_reviews=self.INCLUDE_REVIEWS, + language=language, + ) csv_builder.export_to(path, dataview=dataview) def get_default_language(self, languages): - language = self.dd.default_language - if languages and \ - ((language and language not in languages) or not language): + """Return the default languange of the XForm.""" + language = self.data_dicionary.default_language + if languages and ((language and language not in languages) or not language): languages.sort() language = languages[0] return language def _get_sav_value_labels(self, xpath_var_names=None): - """ GET/SET SPSS `VALUE LABELS`. It takes the dictionary of the form + """GET/SET SPSS `VALUE LABELS`. It takes the dictionary of the form `{varName: {value: valueLabel}}`: .. code-block: python @@ -1009,54 +1281,46 @@ def _get_sav_value_labels(self, xpath_var_names=None): 'available': {0: 'No', 1: 'Yes'} } """ - choice_questions = self.dd.get_survey_elements_with_choices() + choice_questions = self.data_dicionary.get_survey_elements_with_choices() sav_value_labels = {} - for q in choice_questions: - if (xpath_var_names and - q.get_abbreviated_xpath() not in xpath_var_names): + for question in choice_questions: + if ( + xpath_var_names + and question.get_abbreviated_xpath() not in xpath_var_names + ): continue - var_name = xpath_var_names.get(q.get_abbreviated_xpath()) if \ - xpath_var_names else q['name'] - choices = q.to_json_dict().get('children') + var_name = ( + xpath_var_names.get(question.get_abbreviated_xpath()) + if xpath_var_names + else question["name"] + ) + choices = question.to_json_dict().get("children") if choices is None: - choices = self.survey.get('choices') - if choices is not None and q.get('itemset'): - choices = choices.get(q.get('itemset')) + choices = self.survey.get("choices") + if choices is not None and question.get("itemset"): + choices = choices.get(question.get("itemset")) _value_labels = {} if choices: - is_numeric = is_all_numeric([c['name'] for c in choices]) + is_numeric = is_all_numeric([c["name"] for c in choices]) for choice in choices: - name = choice['name'].strip() + name = choice["name"].strip() # should skip select multiple and zero padded numbers e.g # 009 or 09, they should be treated as strings - if q.type != 'select all that apply' and is_numeric: + if question.type != "select all that apply" and is_numeric: try: - name = float(name) \ - if (float(name) > int(name)) else int(name) + name = ( + float(name) if (float(name) > int(name)) else int(name) + ) except ValueError: pass - label = self.get_choice_label_from_dict( - choice.get("label", "")) + label = self.get_choice_label_from_dict(choice.get("label", "")) _value_labels[name] = label.strip() - sav_value_labels[var_name or q['name']] = _value_labels + sav_value_labels[var_name or question["name"]] = _value_labels return sav_value_labels - def _get_var_name(self, title, var_names): - """ - GET valid SPSS varName. - @param title - survey element title/name - @param var_names - list of existing var_names - @return valid varName and list of var_names with new var name appended - """ - var_name = title.replace('/', '.').replace('-', '_'). \ - replace(':', '_').replace('{', '').replace('}', '') - var_name = self._check_sav_column(var_name, var_names) - var_name = '@' + var_name if var_name.startswith('_') else var_name - var_names.append(var_name) - return var_name, var_names - + # pylint: disable=too-many-locals def _get_sav_options(self, elements): """ GET/SET SPSS options. @@ -1071,11 +1335,12 @@ def _get_sav_options(self, elements): 'ioUtf8': True } """ + def _is_numeric(xpath, element_type, data_dictionary): var_name = xpath_var_names.get(xpath) or xpath - if element_type in ['decimal', 'int', 'date']: + if element_type in ["decimal", "int", "date"]: return True - elif element_type == 'string': + if element_type == "string": # check if it is a choice part of multiple choice # type is likely empty string, split multi select is binary element = data_dictionary.get_element(xpath) @@ -1087,13 +1352,12 @@ def _is_numeric(xpath, element_type, data_dictionary): if len(choices) == 0: return False return is_all_numeric(choices) - if element and element.type == '' and value_select_multiples: + if element and element.type == "" and value_select_multiples: return is_all_numeric([element.name]) - parent_xpath = '/'.join(xpath.split('/')[:-1]) + parent_xpath = "/".join(xpath.split("/")[:-1]) parent = data_dictionary.get_element(parent_xpath) - return (parent and parent.type == MULTIPLE_SELECT_TYPE) - else: - return False + return parent and parent.type == MULTIPLE_SELECT_TYPE + return False value_select_multiples = self.VALUE_SELECT_MULTIPLES _var_types = {} @@ -1102,24 +1366,26 @@ def _is_numeric(xpath, element_type, data_dictionary): var_names = [] fields_and_labels = [] - elements += [{'title': f, "label": f, "xpath": f, 'type': f} - for f in self.extra_columns] + elements += [ + {"title": f, "label": f, "xpath": f, "type": f} for f in self.extra_columns + ] for element in elements: - title = element['title'] - _var_name, _var_names = self._get_var_name(title, var_names) + title = element["title"] + _var_name, _var_names = _get_var_name(title, var_names) var_names = _var_names - fields_and_labels.append((element['title'], element['label'], - element['xpath'], _var_name)) + fields_and_labels.append( + (element["title"], element["label"], element["xpath"], _var_name) + ) - xpath_var_names = dict([(xpath, var_name) - for field, label, xpath, var_name - in fields_and_labels]) + xpath_var_names = { + xpath: var_name for field, label, xpath, var_name in fields_and_labels + } all_value_labels = self._get_sav_value_labels(xpath_var_names) duplicate_names = [] # list of (xpath, var_name) already_done = [] # list of xpaths - for field, label, xpath, var_name in fields_and_labels: + for _field, label, xpath, var_name in fields_and_labels: var_labels[var_name] = label # keep track of duplicates if xpath not in already_done: @@ -1138,90 +1404,100 @@ def _get_element_type(element_xpath): return "" var_types = dict( - [(_var_types[element['xpath']], - SAV_NUMERIC_TYPE if _is_numeric(element['xpath'], - element['type'], - self.dd) else - SAV_255_BYTES_TYPE) - for element in elements] + - [(_var_types[item], - SAV_NUMERIC_TYPE if item in [ - '_id', '_index', '_parent_index', SUBMISSION_TIME] - else SAV_255_BYTES_TYPE) - for item in self.extra_columns] + - [(x[1], - SAV_NUMERIC_TYPE if _is_numeric( - x[0], - _get_element_type(x[0]), - self.dd) else SAV_255_BYTES_TYPE) - for x in duplicate_names] + [ + ( + _var_types[element["xpath"]], + SAV_NUMERIC_TYPE + if _is_numeric( + element["xpath"], element["type"], self.data_dicionary + ) + else SAV_255_BYTES_TYPE, + ) + for element in elements + ] + + [ + ( + _var_types[item], + SAV_NUMERIC_TYPE + if item in ["_id", "_index", "_parent_index", SUBMISSION_TIME] + else SAV_255_BYTES_TYPE, + ) + for item in self.extra_columns + ] + + [ + ( + x[1], + SAV_NUMERIC_TYPE + if _is_numeric(x[0], _get_element_type(x[0]), self.data_dicionary) + else SAV_255_BYTES_TYPE, + ) + for x in duplicate_names + ] ) - dates = [_var_types[element['xpath']] for element in elements - if element.get('type') == 'date'] - formats = {d: 'EDATE40' for d in dates} - formats['@' + SUBMISSION_TIME] = 'DATETIME40' + dates = [ + _var_types[element["xpath"]] + for element in elements + if element.get("type") == "date" + ] + formats = {d: "EDATE40" for d in dates} + formats["@" + SUBMISSION_TIME] = "DATETIME40" return { - 'formats': formats, - 'varLabels': var_labels, - 'varNames': var_names, - 'varTypes': var_types, - 'valueLabels': value_labels, - 'ioUtf8': True + "formats": formats, + "varLabels": var_labels, + "varNames": var_names, + "varTypes": var_types, + "valueLabels": value_labels, + "ioUtf8": True, } - def _check_sav_column(self, column, columns): - """ - Check for duplicates and append @ 4 chars uuid. - Also checks for column length more than 64 chars - :param column: - :return: truncated column - """ - - if len(column) > 64: - col_len_diff = len(column) - 64 - column = column[:-col_len_diff] - - if column.lower() in (t.lower() for t in columns): - if len(column) > 59: - column = column[:-5] - column = column + '@' + text(uuid.uuid4()).split('-')[1] - - return column - + # pylint: disable=too-many-locals def to_zipped_sav(self, path, data, *args, **kwargs): - total_records = kwargs.get('total_records') + """Generates the SPSS zipped file format export.""" + if SavWriter is None: + # Fail silently + return - def write_row(row, csv_writer, fields): + total_records = kwargs.get("total_records") + + def write_row(row, sav_writer, fields): # replace character for osm fields - fields = [field.replace(':', '_') for field in fields] + fields = [field.replace(":", "_") for field in fields] sav_writer.writerow( - [encode_if_str(row, field, sav_writer=sav_writer) - for field in fields]) + [encode_if_str(row, field, sav_writer=sav_writer) for field in fields] + ) sav_defs = {} # write headers for section in self.sections: - sav_options = self._get_sav_options(section['elements']) - sav_file = NamedTemporaryFile(suffix='.sav') - sav_writer = SavWriter(sav_file.name, ioLocale=str('en_US.UTF-8'), - **sav_options) - sav_defs[section['name']] = { - 'sav_file': sav_file, 'sav_writer': sav_writer} + sav_options = self._get_sav_options(section["elements"]) + sav_file = NamedTemporaryFile(suffix=".sav") + sav_writer = SavWriter( + sav_file.name, ioLocale=str("en_US.UTF-8"), **sav_options + ) + sav_defs[section["name"]] = {"sav_file": sav_file, "sav_writer": sav_writer} - media_xpaths = [] if not self.INCLUDE_IMAGES \ - else self.dd.get_media_survey_xpaths() + media_xpaths = ( + [] + if not self.INCLUDE_IMAGES + else self.data_dicionary.get_media_survey_xpaths() + ) index = 1 indices = {} survey_name = self.survey.name - for i, d in enumerate(data, start=1): + for i, row_data in enumerate(data, start=1): # decode mongo section names - joined_export = dict_to_joined_export(d, index, indices, - survey_name, - self.survey, d, - media_xpaths) + joined_export = dict_to_joined_export( + row_data, + index, + indices, + survey_name, + self.survey, + row_data, + media_xpaths, + ) output = decode_mongo_encoded_section_names(joined_export) # attach meta fields (index, parent_index, parent_table) # output has keys for every section @@ -1231,52 +1507,53 @@ def write_row(row, csv_writer, fields): output[survey_name][PARENT_INDEX] = -1 for section in self.sections: # get data for this section and write to csv - section_name = section['name'] + section_name = section["name"] sav_def = sav_defs[section_name] - fields = [ - element['xpath'] for element in - section['elements']] - sav_writer = sav_def['sav_writer'] + fields = [element["xpath"] for element in section["elements"]] + sav_writer = sav_def["sav_writer"] row = output.get(section_name, None) if isinstance(row, dict): - write_row( - self.pre_process_row(row, section), - sav_writer, fields) + write_row(self.pre_process_row(row, section), sav_writer, fields) elif isinstance(row, list): for child_row in row: write_row( - self.pre_process_row(child_row, section), - sav_writer, fields) + self.pre_process_row(child_row, section), sav_writer, fields + ) index += 1 track_task_progress(i, total_records) for (section_name, sav_def) in iteritems(sav_defs): - sav_def['sav_writer'].closeSavFile( - sav_def['sav_writer'].fh, mode='wb') + sav_def["sav_writer"].closeSavFile(sav_def["sav_writer"].fh, mode="wb") # write zipfile - with ZipFile(path, 'w', ZIP_DEFLATED, allowZip64=True) as zip_file: + with ZipFile(path, "w", ZIP_DEFLATED, allowZip64=True) as zip_file: for (section_name, sav_def) in iteritems(sav_defs): - sav_file = sav_def['sav_file'] + sav_file = sav_def["sav_file"] sav_file.seek(0) zip_file.write( - sav_file.name, '_'.join(section_name.split('/')) + '.sav') + sav_file.name, "_".join(section_name.split("/")) + ".sav" + ) # close files when we are done for (section_name, sav_def) in iteritems(sav_defs): - sav_def['sav_file'].close() + sav_def["sav_file"].close() def get_fields(self, dataview, section, key): """ Return list of element value with the key in section['elements']. """ if dataview: - return [element.get('_label_xpath') or element[key] - if self.SHOW_CHOICE_LABELS else element[key] - for element in section['elements'] - if element['title'] in dataview.columns] + \ - self.extra_columns - - return [element.get('_label_xpath') or element[key] - if self.SHOW_CHOICE_LABELS else element[key] - for element in section['elements']] + self.extra_columns + return [ + element.get("_label_xpath") or element[key] + if self.SHOW_CHOICE_LABELS + else element[key] + for element in section["elements"] + if element["title"] in dataview.columns + ] + self.extra_columns + + return [ + element.get("_label_xpath") or element[key] + if self.SHOW_CHOICE_LABELS + else element[key] + for element in section["elements"] + ] + self.extra_columns diff --git a/onadata/libs/utils/export_tools.py b/onadata/libs/utils/export_tools.py index a05b32e60c..55c30cdf34 100644 --- a/onadata/libs/utils/export_tools.py +++ b/onadata/libs/utils/export_tools.py @@ -11,50 +11,55 @@ import sys from datetime import datetime, timedelta -import builtins -import six from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.files.base import File from django.core.files.storage import default_storage from django.core.files.temp import NamedTemporaryFile from django.db.models.query import QuerySet from django.shortcuts import render from django.utils import timezone -from django.utils.translation import ugettext as _ -from future.moves.urllib.parse import urlparse -from future.utils import iteritems +from django.utils.translation import gettext as _ + +import six from json2xlsclient.client import Client -from rest_framework import exceptions -from savReaderWriter import SPSSIOError from multidb.pinning import use_master +from rest_framework import exceptions +from six import iteritems +from six.moves.urllib.parse import urlparse + +try: + from savReaderWriter import SPSSIOError +except ImportError: + SPSSIOError = Exception from onadata.apps.logger.models import Attachment, Instance, OsmData, XForm from onadata.apps.logger.models.data_view import DataView from onadata.apps.main.models.meta_data import MetaData -from onadata.apps.viewer.models.export import (Export, - get_export_options_query_kwargs) +from onadata.apps.viewer.models.export import Export, get_export_options_query_kwargs from onadata.apps.viewer.models.parsed_instance import query_data from onadata.libs.exceptions import J2XException, NoRecordsFoundError -from onadata.libs.utils.common_tags import (DATAVIEW_EXPORT, - GROUPNAME_REMOVED_FLAG) -from onadata.libs.utils.common_tools import (str_to_bool, - cmp_to_key, - report_exception, - retry) +from onadata.libs.utils.common_tags import DATAVIEW_EXPORT, GROUPNAME_REMOVED_FLAG +from onadata.libs.utils.common_tools import ( + cmp_to_key, + report_exception, + retry, + str_to_bool, +) from onadata.libs.utils.export_builder import ExportBuilder -from onadata.libs.utils.model_tools import (get_columns_with_hxl, - queryset_iterator) +from onadata.libs.utils.model_tools import get_columns_with_hxl, queryset_iterator from onadata.libs.utils.osm import get_combined_osm -from onadata.libs.utils.viewer_tools import (create_attachments_zipfile, - image_urls) +from onadata.libs.utils.viewer_tools import create_attachments_zipfile, image_urls -DEFAULT_GROUP_DELIMITER = '/' -DEFAULT_INDEX_TAGS = ('[', ']') -SUPPORTED_INDEX_TAGS = ('[', ']', '(', ')', '{', '}', '.', '_') -EXPORT_QUERY_KEY = 'query' +DEFAULT_GROUP_DELIMITER = "/" +DEFAULT_INDEX_TAGS = ("[", "]") +SUPPORTED_INDEX_TAGS = ("[", "]", "(", ")", "{", "}", ".", "_") +EXPORT_QUERY_KEY = "query" MAX_RETRIES = 3 +# pylint: disable=invalid-name +User = get_user_model() + def md5hash(string): """ @@ -69,8 +74,10 @@ def get_export_options(options): list of provided options to be saved with each Export object. """ export_options = { - key: value for (key, value) in iteritems(options) - if key in Export.EXPORT_OPTION_FIELDS} + key: value + for (key, value) in iteritems(options) + if key in Export.EXPORT_OPTION_FIELDS + } return export_options @@ -84,9 +91,7 @@ def get_or_create_export(export_id, xform, export_type, options): try: return Export.objects.get(pk=export_id) except Export.DoesNotExist: - if getattr(settings, 'SLAVE_DATABASES', []): - from multidb.pinning import use_master - + if getattr(settings, "SLAVE_DATABASES", []): with use_master: try: return Export.objects.get(pk=export_id) @@ -98,7 +103,7 @@ def get_or_create_export(export_id, xform, export_type, options): # pylint: disable=too-many-locals, too-many-branches, too-many-statements @retry(MAX_RETRIES) -def generate_export(export_type, xform, export_id=None, options=None): +def generate_export(export_type, xform, export_id=None, options=None): # noqa C901 """ Create appropriate export object given the export type. @@ -127,30 +132,32 @@ def generate_export(export_type, xform, export_id=None, options=None): start = options.get("start") export_type_func_map = { - Export.XLS_EXPORT: 'to_xls_export', - Export.CSV_EXPORT: 'to_flat_csv_export', - Export.CSV_ZIP_EXPORT: 'to_zipped_csv', - Export.SAV_ZIP_EXPORT: 'to_zipped_sav', - Export.GOOGLE_SHEETS_EXPORT: 'to_google_sheets', + Export.XLS_EXPORT: "to_xls_export", + Export.CSV_EXPORT: "to_flat_csv_export", + Export.CSV_ZIP_EXPORT: "to_zipped_csv", + Export.SAV_ZIP_EXPORT: "to_zipped_sav", + Export.GOOGLE_SHEETS_EXPORT: "to_google_sheets", } if xform is None: xform = XForm.objects.get( - user__username__iexact=username, id_string__iexact=id_string) + user__username__iexact=username, id_string__iexact=id_string + ) dataview = None if options.get("dataview_pk"): dataview = DataView.objects.get(pk=options.get("dataview_pk")) - records = dataview.query_data(dataview, all_data=True, - filter_query=filter_query) - total_records = dataview.query_data(dataview, - count=True)[0].get('count') + records = dataview.query_data( + dataview, all_data=True, filter_query=filter_query + ) + total_records = dataview.query_data(dataview, count=True)[0].get("count") else: records = query_data(xform, query=filter_query, start=start, end=end) if filter_query: - total_records = query_data(xform, query=filter_query, start=start, - end=end, count=True)[0].get('count') + total_records = query_data( + xform, query=filter_query, start=start, end=end, count=True + )[0].get("count") else: total_records = xform.num_of_submissions @@ -158,59 +165,64 @@ def generate_export(export_type, xform, export_id=None, options=None): records = records.iterator() export_builder = ExportBuilder() - export_builder.TRUNCATE_GROUP_TITLE = True \ - if export_type == Export.SAV_ZIP_EXPORT else remove_group_name + export_builder.TRUNCATE_GROUP_TITLE = ( + True if export_type == Export.SAV_ZIP_EXPORT else remove_group_name + ) export_builder.GROUP_DELIMITER = options.get( "group_delimiter", DEFAULT_GROUP_DELIMITER ) - export_builder.SPLIT_SELECT_MULTIPLES = options.get( - "split_select_multiples", True - ) + export_builder.SPLIT_SELECT_MULTIPLES = options.get("split_select_multiples", True) export_builder.BINARY_SELECT_MULTIPLES = options.get( "binary_select_multiples", False ) - export_builder.INCLUDE_LABELS = options.get('include_labels', False) - include_reviews = options.get('include_reviews', False) - export_builder.INCLUDE_LABELS_ONLY = options.get( - 'include_labels_only', False - ) - export_builder.INCLUDE_HXL = options.get('include_hxl', False) + export_builder.INCLUDE_LABELS = options.get("include_labels", False) + include_reviews = options.get("include_reviews", False) + export_builder.INCLUDE_LABELS_ONLY = options.get("include_labels_only", False) + export_builder.INCLUDE_HXL = options.get("include_hxl", False) - export_builder.INCLUDE_IMAGES \ - = options.get("include_images", settings.EXPORT_WITH_IMAGE_DEFAULT) + export_builder.INCLUDE_IMAGES = options.get( + "include_images", settings.EXPORT_WITH_IMAGE_DEFAULT + ) - export_builder.VALUE_SELECT_MULTIPLES = options.get( - 'value_select_multiples', False) + export_builder.VALUE_SELECT_MULTIPLES = options.get("value_select_multiples", False) export_builder.REPEAT_INDEX_TAGS = options.get( "repeat_index_tags", DEFAULT_INDEX_TAGS ) - export_builder.SHOW_CHOICE_LABELS = options.get('show_choice_labels', - False) + export_builder.SHOW_CHOICE_LABELS = options.get("show_choice_labels", False) - export_builder.language = options.get('language') + export_builder.language = options.get("language") # 'win_excel_utf8' is only relevant for CSV exports - if 'win_excel_utf8' in options and export_type != Export.CSV_EXPORT: - del options['win_excel_utf8'] + if "win_excel_utf8" in options and export_type != Export.CSV_EXPORT: + del options["win_excel_utf8"] export_builder.INCLUDE_REVIEWS = include_reviews - export_builder.set_survey(xform.survey, xform, - include_reviews=include_reviews) + export_builder.set_survey(xform.survey, xform, include_reviews=include_reviews) temp_file = NamedTemporaryFile(suffix=("." + extension)) columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl( - xform.survey_elements) + xform.survey_elements + ) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) + # pylint: disable=broad-except try: func.__call__( - temp_file.name, records, username, id_string, filter_query, - start=start, end=end, dataview=dataview, xform=xform, - options=options, columns_with_hxl=columns_with_hxl, - total_records=total_records + temp_file.name, + records, + username, + id_string, + filter_query, + start=start, + end=end, + dataview=dataview, + xform=xform, + options=options, + columns_with_hxl=columns_with_hxl, + total_records=total_records, ) except NoRecordsFoundError: pass @@ -223,14 +235,13 @@ def generate_export(export_type, xform, export_id=None, options=None): return export # generate filename - basename = "%s_%s" % ( - id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")) + basename = f'{id_string}_{datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")}' if remove_group_name: # add 'remove group name' flag to filename - basename = "{}-{}".format(basename, GROUPNAME_REMOVED_FLAG) + basename = f"{basename}-{GROUPNAME_REMOVED_FLAG}" if dataview: - basename = "{}-{}".format(basename, DATAVIEW_EXPORT) + basename = f"{basename}-{DATAVIEW_EXPORT}" filename = basename + "." + extension @@ -238,17 +249,11 @@ def generate_export(export_type, xform, export_id=None, options=None): while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) - file_path = os.path.join( - username, - 'exports', - id_string, - export_type, - filename) + file_path = os.path.join(username, "exports", id_string, export_type, filename) # seek to the beginning as required by storage classes temp_file.seek(0) - export_filename = default_storage.save(file_path, - File(temp_file, file_path)) + export_filename = default_storage.save(file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) @@ -275,20 +280,25 @@ def create_export_object(xform, export_type, options): Return an export object that has not been saved to the database. """ export_options = get_export_options(options) - return Export(xform=xform, export_type=export_type, options=export_options, - created_on=timezone.now()) + return Export( + xform=xform, + export_type=export_type, + options=export_options, + created_on=timezone.now(), + ) -def check_pending_export(xform, export_type, options, - minutes=getattr(settings, 'PENDING_EXPORT_TIME', 5)): +def check_pending_export( + xform, export_type, options, minutes=getattr(settings, "PENDING_EXPORT_TIME", 5) +): """ - Check for pending export done within a specific period of time and - returns the export - :param xform: - :param export_type: - :param options: - :param minutes - :return: + Check for pending export done within a specific period of time and + returns the export + :param xform: + :param export_type: + :param options: + :param minutes + :return: """ created_time = timezone.now() - timedelta(minutes=minutes) export_options_kwargs = get_export_options_query_kwargs(options) @@ -297,16 +307,13 @@ def check_pending_export(xform, export_type, options, export_type=export_type, internal_status=Export.PENDING, created_on__gt=created_time, - **export_options_kwargs + **export_options_kwargs, ).last() return export -def should_create_new_export(xform, - export_type, - options, - request=None): +def should_create_new_export(xform, export_type, options, request=None): """ Function that determines whether to create a new export. param: xform @@ -319,27 +326,27 @@ def should_create_new_export(xform, index_tag: ('[', ']') or ('_', '_') params: request: Get params are used to determine if new export is required """ - split_select_multiples = options.get('split_select_multiples', True) + split_select_multiples = options.get("split_select_multiples", True) - if getattr(settings, 'SHOULD_ALWAYS_CREATE_NEW_EXPORT', False): + if getattr(settings, "SHOULD_ALWAYS_CREATE_NEW_EXPORT", False): return True - if (request and (frozenset(list(request.GET)) & - frozenset(['start', 'end', 'data_id']))) or\ - not split_select_multiples: + if ( + request + and (frozenset(list(request.GET)) & frozenset(["start", "end", "data_id"])) + ) or not split_select_multiples: return True export_options_kwargs = get_export_options_query_kwargs(options) export_query = Export.objects.filter( - xform=xform, - export_type=export_type, - **export_options_kwargs + xform=xform, export_type=export_type, **export_options_kwargs ) if options.get(EXPORT_QUERY_KEY) is None: export_query = export_query.exclude(options__has_key=EXPORT_QUERY_KEY) - if export_query.count() == 0 or\ - Export.exports_outdated(xform, export_type, options=options): + if export_query.count() == 0 or Export.exports_outdated( + xform, export_type, options=options + ): return True return False @@ -361,12 +368,10 @@ def newest_export_for(xform, export_type, options): export_options_kwargs = get_export_options_query_kwargs(options) export_query = Export.objects.filter( - xform=xform, - export_type=export_type, - **export_options_kwargs + xform=xform, export_type=export_type, **export_options_kwargs ) - return export_query.latest('created_on') + return export_query.latest("created_on") def increment_index_in_filename(filename): @@ -385,14 +390,15 @@ def increment_index_in_filename(filename): index = 1 # split filename from ext basename, ext = os.path.splitext(filename) - new_filename = "%s-%d%s" % (basename, index, ext) + new_filename = f"{basename}-{index}{ext}" + return new_filename -# pylint: disable=R0913 -def generate_attachments_zip_export(export_type, username, id_string, - export_id=None, options=None, - xform=None): +# pylint: disable=too-many-arguments +def generate_attachments_zip_export( + export_type, username, id_string, export_id=None, options=None, xform=None +): """ Generates zip export of attachments. @@ -413,50 +419,40 @@ def generate_attachments_zip_export(export_type, username, id_string, dataview = DataView.objects.get(pk=options.get("dataview_pk")) attachments = Attachment.objects.filter( instance_id__in=[ - rec.get('_id') + rec.get("_id") for rec in dataview.query_data( - dataview, all_data=True, filter_query=filter_query)], - instance__deleted_at__isnull=True) + dataview, all_data=True, filter_query=filter_query + ) + ], + instance__deleted_at__isnull=True, + ) else: instance_ids = query_data(xform, fields='["_id"]', query=filter_query) - attachments = Attachment.objects.filter( - instance__deleted_at__isnull=True) + attachments = Attachment.objects.filter(instance__deleted_at__isnull=True) if xform.is_merged_dataset: attachments = attachments.filter( - instance__xform_id__in=[ - i for i in xform.mergedxform.xforms.filter( - deleted_at__isnull=True).values_list( - 'id', flat=True)]).filter( - instance_id__in=[i_id['_id'] for i_id in instance_ids]) + instance__xform_id__in=list( + xform.mergedxform.xforms.filter( + deleted_at__isnull=True + ).values_list("id", flat=True) + ) + ).filter(instance_id__in=[i_id["_id"] for i_id in instance_ids]) else: - attachments = attachments.filter( - instance__xform_id=xform.pk).filter( - instance_id__in=[i_id['_id'] for i_id in instance_ids]) - - filename = "%s_%s.%s" % (id_string, - datetime.now().strftime("%Y_%m_%d_%H_%M_%S"), - export_type.lower()) - file_path = os.path.join( - username, - 'exports', - id_string, - export_type, - filename) - zip_file = None + attachments = attachments.filter(instance__xform_id=xform.pk).filter( + instance_id__in=[i_id["_id"] for i_id in instance_ids] + ) - try: - zip_file = create_attachments_zipfile(attachments) + filename = ( + f'{id_string}_{datetime.now().strftime("%Y_%m_%d_%H_%M_%S")}' + f".{export_type.lower()}" + ) + file_path = os.path.join(username, "exports", id_string, export_type, filename) + zip_file = None - try: - temp_file = builtins.open(zip_file.name, 'rb') - filename = default_storage.save( - file_path, - File(temp_file, file_path)) - finally: - temp_file.close() - finally: - if zip_file: - zip_file.close() + with NamedTemporaryFile() as zip_file: + create_attachments_zipfile(attachments, zip_file) + with open(zip_file.name, "rb") as temp_file: + filename = default_storage.save(file_path, File(temp_file, file_path)) export = get_or_create_export(export_id, xform, export_type, options) export.filedir, export.filename = os.path.split(filename) @@ -467,7 +463,7 @@ def generate_attachments_zip_export(export_type, username, id_string, def write_temp_file_to_path(suffix, content, file_path): - """ Write a temp file and return the name of the file. + """Write a temp file and return the name of the file. :param suffix: The file suffix :param content: The content to write :param file_path: The path to write the temp file to @@ -476,16 +472,14 @@ def write_temp_file_to_path(suffix, content, file_path): temp_file = NamedTemporaryFile(suffix=suffix) temp_file.write(content) temp_file.seek(0) - export_filename = default_storage.save( - file_path, - File(temp_file, file_path)) + export_filename = default_storage.save(file_path, File(temp_file, file_path)) temp_file.close() return export_filename def get_or_create_export_object(export_id, options, xform, export_type): - """ Get or create export object. + """Get or create export object. :param export_id: Export ID :param options: Options to convert to export options @@ -494,19 +488,27 @@ def get_or_create_export_object(export_id, options, xform, export_type): :return: A new or found export object """ if export_id and Export.objects.filter(pk=export_id).exists(): - export = Export.objects.get(id=export_id) + try: + export = Export.objects.get(id=export_id) + except Export.DoesNotExist: + with use_master: + try: + return Export.objects.get(pk=export_id) + except Export.DoesNotExist: + pass else: export_options = get_export_options(options) - export = Export.objects.create(xform=xform, - export_type=export_type, - options=export_options) + export = Export.objects.create( + xform=xform, export_type=export_type, options=export_options + ) return export -# pylint: disable=R0913 -def generate_kml_export(export_type, username, id_string, export_id=None, - options=None, xform=None): +# pylint: disable=too-many-arguments +def generate_kml_export( + export_type, username, id_string, export_id=None, options=None, xform=None +): """ Generates kml export for geographical data @@ -524,25 +526,18 @@ def generate_kml_export(export_type, username, id_string, export_id=None, xform = XForm.objects.get(user__username=username, id_string=id_string) response = render( - None, 'survey.kml', - {'data': kml_export_data(id_string, user, xform=xform)} + None, "survey.kml", {"data": kml_export_data(id_string, user, xform=xform)} ) - basename = "%s_%s" % (id_string, - datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) + basename = f'{id_string}_{datetime.now().strftime("%Y_%m_%d_%H_%M_%S")}' filename = basename + "." + export_type.lower() - file_path = os.path.join( - username, - 'exports', - id_string, - export_type, - filename) + file_path = os.path.join(username, "exports", id_string, export_type, filename) export_filename = write_temp_file_to_path( - export_type.lower(), response.content, file_path) + export_type.lower(), response.content, file_path + ) - export = get_or_create_export_object( - export_id, options, xform, export_type) + export = get_or_create_export_object(export_id, options, xform, export_type) export.filedir, export.filename = os.path.split(export_filename) export.internal_status = Export.SUCCESSFUL @@ -555,6 +550,7 @@ def kml_export_data(id_string, user, xform=None): """ KML export data from form submissions. """ + def cached_get_labels(xpath): """ Get and Cache labels for the XForm. @@ -567,16 +563,20 @@ def cached_get_labels(xpath): xform = xform or XForm.objects.get(id_string=id_string, user=user) - data_kwargs = {'geom__isnull': False} + data_kwargs = {"geom__isnull": False} if xform.is_merged_dataset: - data_kwargs.update({ - 'xform_id__in': - [i for i in xform.mergedxform.xforms.filter( - deleted_at__isnull=True).values_list('id', flat=True)] - }) + data_kwargs.update( + { + "xform_id__in": list( + xform.mergedxform.xforms.filter( + deleted_at__isnull=True + ).values_list("id", flat=True) + ) + } + ) else: - data_kwargs.update({'xform_id': xform.pk}) - instances = Instance.objects.filter(**data_kwargs).order_by('id') + data_kwargs.update({"xform_id": xform.pk}) + instances = Instance.objects.filter(**data_kwargs).order_by("id") data_for_template = [] labels = {} for instance in queryset_iterator(instances): @@ -585,22 +585,28 @@ def cached_get_labels(xpath): xpaths = list(data_for_display) xpaths.sort(key=cmp_to_key(instance.xform.get_xpath_cmp())) table_rows = [ - '%s%s' % - (cached_get_labels(xpath), data_for_display[xpath]) for xpath in - xpaths if not xpath.startswith(u"_")] + f"{cached_get_labels(xpath) }" + f"{data_for_display[xpath]}" + for xpath in xpaths + if not xpath.startswith("_") + ] img_urls = image_urls(instance) if instance.point: - data_for_template.append({ - 'name': instance.xform.id_string, - 'id': instance.id, - 'lat': instance.point.y, - 'lng': instance.point.x, - 'image_urls': img_urls, - 'table': '%s' - '
    ' % (img_urls[0] if img_urls else "", - ''.join(table_rows))}) + img_url = img_urls[0] if img_urls else "" + rows = "".join(table_rows) + data_for_template.append( + { + "name": instance.xform.id_string, + "id": instance.id, + "lat": instance.point.y, + "lng": instance.point.x, + "image_urls": img_urls, + "table": '{rows}' + "
    ", + } + ) return data_for_template @@ -608,20 +614,23 @@ def cached_get_labels(xpath): def get_osm_data_kwargs(xform): """Return kwargs for OsmData queryset for given xform""" - kwargs = {'instance__deleted_at__isnull': True} + kwargs = {"instance__deleted_at__isnull": True} if xform.is_merged_dataset: - kwargs['instance__xform_id__in'] = [ - i for i in xform.mergedxform.xforms.filter( - deleted_at__isnull=True).values_list('id', flat=True)] + kwargs["instance__xform_id__in"] = list( + xform.mergedxform.xforms.filter(deleted_at__isnull=True).values_list( + "id", flat=True + ) + ) else: - kwargs['instance__xform_id'] = xform.pk + kwargs["instance__xform_id"] = xform.pk return kwargs -def generate_osm_export(export_type, username, id_string, export_id=None, - options=None, xform=None): +def generate_osm_export( + export_type, username, id_string, export_id=None, options=None, xform=None +): """ Generates osm export for OpenStreetMap data @@ -641,21 +650,14 @@ def generate_osm_export(export_type, username, id_string, export_id=None, kwargs = get_osm_data_kwargs(xform) osm_list = OsmData.objects.filter(**kwargs) content = get_combined_osm(osm_list) - - basename = "%s_%s" % (id_string, - datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) + timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S") + basename = f"{id_string}_{timestamp}" filename = basename + "." + extension - file_path = os.path.join( - username, - 'exports', - id_string, - export_type, - filename) + file_path = os.path.join(username, "exports", id_string, export_type, filename) export_filename = write_temp_file_to_path(extension, content, file_path) - export = get_or_create_export_object( - export_id, options, xform, export_type) + export = get_or_create_export_object(export_id, options, xform, export_type) dir_name, basename = os.path.split(export_filename) export.filedir = dir_name @@ -667,8 +669,7 @@ def generate_osm_export(export_type, username, id_string, export_id=None, def _get_records(instances): - return [clean_keys_of_slashes(instance) - for instance in instances] + return [clean_keys_of_slashes(instance) for instance in instances] def clean_keys_of_slashes(record): @@ -679,10 +680,9 @@ def clean_keys_of_slashes(record): """ for key in list(record): value = record[key] - if '/' in key: + if "/" in key: # replace with _ - record[key.replace('/', '_')]\ - = record.pop(key) + record[key.replace("/", "_")] = record.pop(key) # Check if the value is a list containing nested dict and apply same if value: if isinstance(value, list) and isinstance(value[0], dict): @@ -697,15 +697,14 @@ def _get_server_from_metadata(xform, meta, token): try: report_templates = MetaData.external_export(xform) except MetaData.DoesNotExist: - from multidb.pinning import use_master with use_master: report_templates = MetaData.external_export(xform) if meta: try: int(meta) - except ValueError: - raise Exception(u"Invalid metadata pk {0}".format(meta)) + except ValueError as e: + raise Exception(f"Invalid metadata pk {meta}") from e # Get the external server from the metadata result = report_templates.get(pk=meta) @@ -718,7 +717,8 @@ def _get_server_from_metadata(xform, meta, token): # Take the latest value in the metadata if not report_templates: raise Exception( - u"Could not find the template token: Please upload template.") + "Could not find the template token: Please upload template." + ) server = report_templates[0].external_export_url name = report_templates[0].external_export_name @@ -726,8 +726,9 @@ def _get_server_from_metadata(xform, meta, token): return server, name -def generate_external_export(export_type, username, id_string, export_id=None, - options=None, xform=None): +def generate_external_export( # noqa C901 + export_type, username, id_string, export_id=None, options=None, xform=None +): """ Generates external export using ONA data through an external service. @@ -748,7 +749,8 @@ def generate_external_export(export_type, username, id_string, export_id=None, if xform is None: xform = XForm.objects.get( - user__username__iexact=username, id_string__iexact=id_string) + user__username__iexact=username, id_string__iexact=id_string + ) user = User.objects.get(username=username) server, name = _get_server_from_metadata(xform, meta, token) @@ -758,14 +760,13 @@ def generate_external_export(export_type, username, id_string, export_id=None, token = parsed_url.path[5:] - ser = parsed_url.scheme + '://' + parsed_url.netloc + ser = parsed_url.scheme + "://" + parsed_url.netloc # Get single submission data if data_id: - inst = Instance.objects.filter(xform__user=user, - xform__id_string=id_string, - deleted_at=None, - pk=data_id) + inst = Instance.objects.filter( + xform__user=user, xform__id_string=id_string, deleted_at=None, pk=data_id + ) instances = [inst[0].json if inst else {}] else: @@ -780,20 +781,18 @@ def generate_external_export(export_type, username, id_string, export_id=None, client = Client(ser) response = client.xls.create(token, json.dumps(records)) - if hasattr(client.xls.conn, 'last_response'): + if hasattr(client.xls.conn, "last_response"): status_code = client.xls.conn.last_response.status_code except Exception as e: raise J2XException( - u"J2X client could not generate report. Server -> {0}," - u" Error-> {1}".format(server, e) - ) + f"J2X client could not generate report. Server -> {server}," + f" Error-> {e}" + ) from e else: if not server: - raise J2XException(u"External server not set") - elif not records: - raise J2XException( - u"No record to export. Form -> {0}".format(id_string) - ) + raise J2XException("External server not set") + if not records: + raise J2XException(f"No record to export. Form -> {id_string}") # get or create export object if export_id: @@ -804,14 +803,14 @@ def generate_external_export(export_type, username, id_string, export_id=None, export = Export.objects.get(id=export_id) else: export_options = get_export_options(options) - export = Export.objects.create(xform=xform, - export_type=export_type, - options=export_options) + export = Export.objects.create( + xform=xform, export_type=export_type, options=export_options + ) export.export_url = response if status_code == 201: export.internal_status = Export.SUCCESSFUL - export.filename = name + '-' + response[5:] if name else response[5:] + export.filename = name + "-" + response[5:] if name else response[5:] export.export_url = ser + response else: export.internal_status = Export.FAILED @@ -832,84 +831,87 @@ def upload_template_for_external_export(server, file_obj): response = client.template.create(template_file=file_obj) status_code = None - if hasattr(client.template.conn, 'last_response'): + if hasattr(client.template.conn, "last_response"): status_code = client.template.conn.last_response.status_code - return str(status_code) + '|' + response + return str(status_code) + "|" + response -def parse_request_export_options(params): # pylint: disable=too-many-branches +# pylint: disable=too-many-branches +def parse_request_export_options(params): # noqa C901 """ Parse export options in the request object into values returned in a list. The list represents a boolean for whether the group name should be removed, the group delimiter, and a boolean for whether select multiples should be split. """ - boolean_list = ['true', 'false'] + boolean_list = ["true", "false"] options = {} - remove_group_name = params.get('remove_group_name') and \ - params.get('remove_group_name').lower() - binary_select_multiples = params.get('binary_select_multiples') and \ - params.get('binary_select_multiples').lower() - do_not_split_select_multiples = params.get( - 'do_not_split_select_multiples') - include_labels = params.get('include_labels', False) - include_reviews = params.get('include_reviews', False) - include_labels_only = params.get('include_labels_only', False) - include_hxl = params.get('include_hxl', True) - value_select_multiples = params.get('value_select_multiples') and \ - params.get('value_select_multiples').lower() - show_choice_labels = params.get('show_choice_labels') and \ - params.get('show_choice_labels').lower() + remove_group_name = ( + params.get("remove_group_name") and params.get("remove_group_name").lower() + ) + binary_select_multiples = ( + params.get("binary_select_multiples") + and params.get("binary_select_multiples").lower() + ) + do_not_split_select_multiples = params.get("do_not_split_select_multiples") + include_labels = params.get("include_labels", False) + include_reviews = params.get("include_reviews", False) + include_labels_only = params.get("include_labels_only", False) + include_hxl = params.get("include_hxl", True) + value_select_multiples = ( + params.get("value_select_multiples") + and params.get("value_select_multiples").lower() + ) + show_choice_labels = ( + params.get("show_choice_labels") and params.get("show_choice_labels").lower() + ) if include_labels is not None: - options['include_labels'] = str_to_bool(include_labels) + options["include_labels"] = str_to_bool(include_labels) if include_reviews is not None: - options['include_reviews'] = str_to_bool(include_reviews) + options["include_reviews"] = str_to_bool(include_reviews) if include_labels_only is not None: - options['include_labels_only'] = str_to_bool(include_labels_only) + options["include_labels_only"] = str_to_bool(include_labels_only) if include_hxl is not None: - options['include_hxl'] = str_to_bool(include_hxl) + options["include_hxl"] = str_to_bool(include_hxl) if remove_group_name in boolean_list: options["remove_group_name"] = str_to_bool(remove_group_name) else: options["remove_group_name"] = False - if params.get("group_delimiter") in ['.', DEFAULT_GROUP_DELIMITER]: - options['group_delimiter'] = params.get("group_delimiter") + if params.get("group_delimiter") in [".", DEFAULT_GROUP_DELIMITER]: + options["group_delimiter"] = params.get("group_delimiter") else: - options['group_delimiter'] = DEFAULT_GROUP_DELIMITER + options["group_delimiter"] = DEFAULT_GROUP_DELIMITER - options['split_select_multiples'] = \ - not str_to_bool(do_not_split_select_multiples) + options["split_select_multiples"] = not str_to_bool(do_not_split_select_multiples) if binary_select_multiples and binary_select_multiples in boolean_list: - options['binary_select_multiples'] = str_to_bool( - binary_select_multiples) + options["binary_select_multiples"] = str_to_bool(binary_select_multiples) - if 'include_images' in params: - options["include_images"] = str_to_bool( - params.get("include_images")) + if "include_images" in params: + options["include_images"] = str_to_bool(params.get("include_images")) else: options["include_images"] = settings.EXPORT_WITH_IMAGE_DEFAULT - options['win_excel_utf8'] = str_to_bool(params.get('win_excel_utf8')) + options["win_excel_utf8"] = str_to_bool(params.get("win_excel_utf8")) if value_select_multiples and value_select_multiples in boolean_list: - options['value_select_multiples'] = str_to_bool(value_select_multiples) + options["value_select_multiples"] = str_to_bool(value_select_multiples) if show_choice_labels and show_choice_labels in boolean_list: - options['show_choice_labels'] = str_to_bool(show_choice_labels) + options["show_choice_labels"] = str_to_bool(show_choice_labels) index_tags = get_repeat_index_tags(params.get("repeat_index_tags")) if index_tags: - options['repeat_index_tags'] = index_tags + options["repeat_index_tags"] = index_tags - if 'language' in params: - options['language'] = params.get('language') + if "language" in params: + options["language"] = params.get("language") return options @@ -921,7 +923,7 @@ def get_repeat_index_tags(index_tags): Retuns a tuple of two strings with SUPPORTED_INDEX_TAGS, """ if isinstance(index_tags, six.string_types): - index_tags = tuple(index_tags.split(',')) + index_tags = tuple(index_tags.split(",")) length = len(index_tags) if length == 1: index_tags = (index_tags[0], index_tags[0]) @@ -932,8 +934,11 @@ def get_repeat_index_tags(index_tags): for tag in index_tags: if tag not in SUPPORTED_INDEX_TAGS: - raise exceptions.ParseError(_( - "The tag %s is not supported, supported tags are %s" % - (tag, SUPPORTED_INDEX_TAGS))) + raise exceptions.ParseError( + _( + f"The tag {tag} is not supported, " + f"supported tags are {SUPPORTED_INDEX_TAGS}" + ) + ) return index_tags diff --git a/onadata/libs/utils/google.py b/onadata/libs/utils/google.py index cb0fb7c0cd..33fb4cd5f9 100644 --- a/onadata/libs/utils/google.py +++ b/onadata/libs/utils/google.py @@ -1,11 +1,22 @@ -from oauth2client.client import OAuth2WebServerFlow +# -*- coding=utf-8 -*- +""" +Google utility functions. +""" +from typing import Optional + from django.conf import settings -google_flow = OAuth2WebServerFlow( - client_id=settings.GOOGLE_OAUTH2_CLIENT_ID, - client_secret=settings.GOOGLE_OAUTH2_CLIENT_SECRET, - scope=' '.join( - ['https://docs.google.com/feeds/', - 'https://spreadsheets.google.com/feeds/', - 'https://www.googleapis.com/auth/drive.file']), - redirect_uri=settings.GOOGLE_STEP2_URI, prompt="consent") +from google_auth_oauthlib.flow import Flow + + +def create_flow(redirect_uri: Optional[str] = None) -> Flow: + """Returns a Google Flow from client configuration.""" + return Flow.from_client_config( + settings.GOOGLE_FLOW, + scopes=[ + "https://www.googleapis.com/auth/spreadsheets", + "https://www.googleapis.com/auth/docs", + "https://www.googleapis.com/auth/drive.file", + ], + redirect_uri=redirect_uri or settings.GOOGLE_STEP2_URI, + ) diff --git a/onadata/libs/utils/gravatar.py b/onadata/libs/utils/gravatar.py index afe3b0a18e..f508e32ed0 100644 --- a/onadata/libs/utils/gravatar.py +++ b/onadata/libs/utils/gravatar.py @@ -1,6 +1,11 @@ +# -*- coding: utf-8 -*- +"""Gravatar utils module""" import hashlib -from future.moves.urllib.parse import urlencode -from future.moves.urllib.request import urlopen + +from six.moves.urllib.parse import urlencode + +import requests + DEFAULT_GRAVATAR = "https://ona.io/static/images/default_avatar.png" GRAVATAR_ENDPOINT = "https://secure.gravatar.com/avatar/" @@ -8,14 +13,23 @@ def email_md5(user): - return hashlib.md5(user.email.lower().encode('utf-8')).hexdigest() + """Returns the hash of an email for the user""" + return hashlib.new( + "md5", user.email.lower().encode("utf-8"), usedforsecurity=False + ).hexdigest() def get_gravatar_img_link(user): - return GRAVATAR_ENDPOINT + email_md5(user) + "?" + urlencode({ - 'd': DEFAULT_GRAVATAR, 's': str(GRAVATAR_SIZE)}) + """Returns the Gravatar image URL""" + return ( + GRAVATAR_ENDPOINT + + email_md5(user) + + "?" + + urlencode({"d": DEFAULT_GRAVATAR, "s": str(GRAVATAR_SIZE)}) + ) def gravatar_exists(user): + """Checks if the Gravatar URL exists""" url = GRAVATAR_ENDPOINT + email_md5(user) + "?" + "d=404" - return urlopen(url).getcode() != 404 + return requests.get(url).status_code != 404 diff --git a/onadata/libs/utils/log.py b/onadata/libs/utils/log.py index 784edc1fba..8b2d470c19 100644 --- a/onadata/libs/utils/log.py +++ b/onadata/libs/utils/log.py @@ -1,7 +1,7 @@ import logging from datetime import datetime -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from onadata.libs.utils.viewer_tools import get_client_ip diff --git a/onadata/libs/utils/logger_tools.py b/onadata/libs/utils/logger_tools.py index 0b236c1838..85e4a588ed 100644 --- a/onadata/libs/utils/logger_tools.py +++ b/onadata/libs/utils/logger_tools.py @@ -1,3 +1,8 @@ +# -*- coding: utf-8 -*- +""" +logger_tools - Logger app utility functions. +""" +import json import os import re import sys @@ -9,99 +14,134 @@ from typing import NoReturn from wsgiref.util import FileWrapper from xml.dom import Node -import xml.etree.ElementTree as ET from xml.parsers.expat import ExpatError -import pytz -from dict2xml import dict2xml from django.conf import settings -from django.contrib.auth.models import User -from django.core.exceptions import (MultipleObjectsReturned, PermissionDenied, - ValidationError) +from django.contrib.auth import get_user_model +from django.core.exceptions import ( + MultipleObjectsReturned, + PermissionDenied, + ValidationError, +) from django.core.files.storage import get_storage_class -from django.db import IntegrityError, transaction, DataError +from django.db import DataError, IntegrityError, transaction from django.db.models import Q -from django.http import (HttpResponse, HttpResponseNotFound, - StreamingHttpResponse, UnreadablePostError) +from django.http import ( + HttpResponse, + HttpResponseNotFound, + StreamingHttpResponse, + UnreadablePostError, +) from django.shortcuts import get_object_or_404 from django.utils import timezone from django.utils.encoding import DjangoUnicodeDecodeError -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ + +import pytz +from defusedxml.ElementTree import fromstring +from dict2xml import dict2xml from modilabs.utils.subprocess_timeout import ProcessTimedOut from multidb.pinning import use_master from pyxform.errors import PyXFormError from pyxform.xform2json import create_survey_element_from_xml from rest_framework.response import Response -from onadata.apps.logger.models import ( - Attachment, Instance, XForm, XFormVersion) +from onadata.apps.logger.models import Attachment, Instance, XForm, XFormVersion from onadata.apps.logger.models.instance import ( - FormInactiveError, InstanceHistory, FormIsMergedDatasetError, - get_id_string_from_xml_str) -from onadata.apps.logger.models.xform import XLSFormError + FormInactiveError, + FormIsMergedDatasetError, + InstanceHistory, + get_id_string_from_xml_str, +) +from onadata.apps.logger.models.xform import DuplicateUUIDError, XLSFormError from onadata.apps.logger.xform_instance_parser import ( - DuplicateInstance, InstanceEmptyError, InstanceInvalidUserError, - InstanceMultipleNodeError, InstanceEncryptionError, NonUniqueFormIdError, - InstanceFormatError, clean_and_parse_xml, get_deprecated_uuid_from_xml, - get_submission_date_from_xml, get_uuid_from_xml, AttachmentNameError) -from onadata.apps.messaging.constants import XFORM, \ - SUBMISSION_EDITED, SUBMISSION_CREATED + AttachmentNameError, + DuplicateInstance, + InstanceEmptyError, + InstanceEncryptionError, + InstanceFormatError, + InstanceInvalidUserError, + InstanceMultipleNodeError, + NonUniqueFormIdError, + clean_and_parse_xml, + get_deprecated_uuid_from_xml, + get_submission_date_from_xml, + get_uuid_from_xml, +) +from onadata.apps.messaging.constants import ( + SUBMISSION_CREATED, + SUBMISSION_EDITED, + XFORM, +) from onadata.apps.messaging.serializers import send_message from onadata.apps.viewer.models.data_dictionary import DataDictionary from onadata.apps.viewer.models.parsed_instance import ParsedInstance from onadata.apps.viewer.signals import process_submission from onadata.libs.utils.analytics import track_object_event from onadata.libs.utils.common_tags import METADATA_FIELDS -from onadata.libs.utils.common_tools import report_exception, get_uuid +from onadata.libs.utils.common_tools import get_uuid, report_exception from onadata.libs.utils.model_tools import set_uuid from onadata.libs.utils.user_auth import get_user_default_project -OPEN_ROSA_VERSION_HEADER = 'X-OpenRosa-Version' -HTTP_OPEN_ROSA_VERSION_HEADER = 'HTTP_X_OPENROSA_VERSION' -OPEN_ROSA_VERSION = '1.0' -DEFAULT_CONTENT_TYPE = 'text/xml; charset=utf-8' +OPEN_ROSA_VERSION_HEADER = "X-OpenRosa-Version" +HTTP_OPEN_ROSA_VERSION_HEADER = "HTTP_X_OPENROSA_VERSION" +OPEN_ROSA_VERSION = "1.0" +DEFAULT_CONTENT_TYPE = "text/xml; charset=utf-8" DEFAULT_CONTENT_LENGTH = settings.DEFAULT_CONTENT_LENGTH REQUIRED_ENCRYPTED_FILE_ELEMENTS = [ "{http://www.opendatakit.org/xforms/encrypted}base64EncryptedKey", "{http://www.opendatakit.org/xforms/encrypted}encryptedXmlFile", "{http://opendatakit.org/submissions}base64EncryptedKey", - "{http://opendatakit.org/submissions}encryptedXmlFile"] + "{http://opendatakit.org/submissions}encryptedXmlFile", +] + +uuid_regex = re.compile( + r"\s*\s*([^<]+)\s*\s*", re.DOTALL +) + -uuid_regex = re.compile(r'\s*\s*([^<]+)\s*\s*', - re.DOTALL) +# pylint: disable=invalid-name +User = get_user_model() def create_xform_version(xform: XForm, user: User) -> XFormVersion: """ Creates an XFormVersion object for the passed in XForm """ + versioned_xform = None try: with transaction.atomic(): - return XFormVersion.objects.create( + versioned_xform = XFormVersion.objects.create( xform=xform, xls=xform.xls, - json=xform.json, + json=xform.json + if isinstance(xform.json, str) + else json.dumps(xform.json), version=xform.version, created_by=user, - xml=xform.xml + xml=xform.xml, ) except IntegrityError: pass + return versioned_xform -def _get_instance(xml, new_uuid, submitted_by, status, xform, checksum, - request=None): +# pylint: disable=too-many-arguments +def _get_instance(xml, new_uuid, submitted_by, status, xform, checksum, request=None): history = None instance = None message_verb = SUBMISSION_EDITED # check if its an edit submission old_uuid = get_deprecated_uuid_from_xml(xml) if old_uuid: - instance = Instance.objects.filter(uuid=old_uuid, - xform_id=xform.pk).first() - history = InstanceHistory.objects.filter( - xform_instance__xform_id=xform.pk, - uuid=new_uuid).only('xform_instance').first() + instance = Instance.objects.filter(uuid=old_uuid, xform_id=xform.pk).first() + history = ( + InstanceHistory.objects.filter( + xform_instance__xform_id=xform.pk, uuid=new_uuid + ) + .only("xform_instance") + .first() + ) if instance: # edits @@ -115,7 +155,8 @@ def _get_instance(xml, new_uuid, submitted_by, status, xform, checksum, uuid=old_uuid, user=submitted_by, geom=instance.geom, - submission_date=instance.last_edited or instance.date_created) + submission_date=instance.last_edited or instance.date_created, + ) instance.xml = xml instance.last_edited = last_edited instance.uuid = new_uuid @@ -123,22 +164,24 @@ def _get_instance(xml, new_uuid, submitted_by, status, xform, checksum, instance.save() # call webhooks - process_submission.send(sender=instance.__class__, - instance=instance) + process_submission.send(sender=instance.__class__, instance=instance) elif history: instance = history.xform_instance if old_uuid is None or (instance is None and history is None): # new submission message_verb = SUBMISSION_CREATED instance = Instance.objects.create( - xml=xml, user=submitted_by, status=status, xform=xform, - checksum=checksum) + xml=xml, user=submitted_by, status=status, xform=xform, checksum=checksum + ) # send notification on submission creation send_message( - instance_id=instance.id, target_id=instance.xform.id, - target_type=XFORM, user=instance.user or instance.xform.user, - message_verb=message_verb) + instance_id=instance.id, + target_id=instance.xform.id, + target_type=XFORM, + user=instance.user or instance.xform.user, + message_verb=message_verb, + ) return instance @@ -168,25 +211,23 @@ def dict2xform(jsform, form_id, root=None, username=None, gen_uuid=False): form = XForm.objects.filter( id_string__iexact=form_id, user__username__iexact=username, - deleted_at__isnull=True).first() - root = form.survey.name if form else 'data' + deleted_at__isnull=True, + ).first() + root = form.survey.name if form else "data" else: - root = 'data' + root = "data" if gen_uuid: - jsform['meta'] = { - 'instanceID': 'uuid:' + get_uuid(hex_only=False) - } + jsform["meta"] = {"instanceID": "uuid:" + get_uuid(hex_only=False)} - return "<{0} id='{1}'>{2}".format( - root, form_id, dict2xml(jsform)) + return f"<{root} id='{form_id}'>{dict2xml(jsform)}" def get_first_record(queryset): """ Returns the first item in a queryset sorted by id. """ - records = sorted([record for record in queryset], key=lambda k: k.id) + records = sorted(list(queryset), key=lambda k: k.id) if records: return records[0] @@ -194,15 +235,15 @@ def get_first_record(queryset): def get_uuid_from_submission(xml): + """Extracts and returns the UUID from a submission XML.""" # parse UUID from uploaded XML - split_xml = uuid_regex.split(xml.decode('utf-8')) + split_xml = uuid_regex.split(xml.decode("utf-8")) # check that xml has UUID - return len(split_xml) > 1 and split_xml[1] or None + return split_xml[1] if len(split_xml) > 1 else None -def get_xform_from_submission( - xml, username, uuid=None, request=None): +def get_xform_from_submission(xml, username, uuid=None, request=None): """Gets the submissions target XForm. Retrieves the target XForm by either utilizing the `uuid` param @@ -223,8 +264,7 @@ def get_xform_from_submission( if uuid: # try find the form by its uuid which is the ideal condition - if XForm.objects.filter( - uuid=uuid, deleted_at__isnull=True).count() > 0: + if XForm.objects.filter(uuid=uuid, deleted_at__isnull=True).count() > 0: xform = XForm.objects.get(uuid=uuid, deleted_at__isnull=True) # If request is present, verify that the request user # has the correct permissions @@ -239,41 +279,47 @@ def get_xform_from_submission( # Assumption: If the owner_username is equal to the XForm # owner we've retrieved the correct form. if username and xform.user.username == username: - raise e + raise e from e else: return xform id_string = get_id_string_from_xml_str(xml) try: return get_object_or_404( - XForm, - id_string__iexact=id_string, - user__username__iexact=username, - deleted_at__isnull=True) - except MultipleObjectsReturned: - raise NonUniqueFormIdError() + XForm, + id_string__iexact=id_string, + user__username__iexact=username, + deleted_at__isnull=True, + ) + except MultipleObjectsReturned as e: + raise NonUniqueFormIdError() from e def _has_edit_xform_permission(xform, user): if isinstance(xform, XForm) and isinstance(user, User): - return user.has_perm('logger.change_xform', xform) + return user.has_perm("logger.change_xform", xform) return False def check_edit_submission_permissions(request_user, xform): + """Checks edit submission permissions.""" if xform and request_user and request_user.is_authenticated: requires_auth = xform.user.profile.require_auth has_edit_perms = _has_edit_xform_permission(xform, request_user) if requires_auth and not has_edit_perms: raise PermissionDenied( - _(u"%(request_user)s is not allowed to make edit submissions " - u"to %(form_user)s's %(form_title)s form." % { - 'request_user': request_user, - 'form_user': xform.user, - 'form_title': xform.title - })) + _( + "%(request_user)s is not allowed to make edit submissions " + "to %(form_user)s's %(form_title)s form." + % { + "request_user": request_user, + "form_user": xform.user, + "form_title": xform.title, + } + ) + ) def check_submission_permissions(request, xform): @@ -290,17 +336,27 @@ def check_submission_permissions(request, xform): :returns: None. :raises: PermissionDenied based on the above criteria. """ - if request and (xform.user.profile.require_auth or xform.require_auth or - request.path == '/submission')\ - and xform.user != request.user\ - and not request.user.has_perm('report_xform', xform): + requires_authentication = request and ( + xform.user.profile.require_auth + or xform.require_auth + or request.path == "/submission" + ) + if ( + requires_authentication + and xform.user != request.user + and not request.user.has_perm("report_xform", xform) + ): raise PermissionDenied( - _(u"%(request_user)s is not allowed to make submissions " - u"to %(form_user)s's %(form_title)s form." % { - 'request_user': request.user, - 'form_user': xform.user, - 'form_title': xform.title - })) + _( + "%(request_user)s is not allowed to make submissions " + "to %(form_user)s's %(form_title)s form." + % { + "request_user": request.user, + "form_user": xform.user, + "form_title": xform.title, + } + ) + ) def check_submission_encryption(xform: XForm, xml: bytes) -> NoReturn: @@ -312,24 +368,27 @@ def check_submission_encryption(xform: XForm, xml: bytes) -> NoReturn: from the submissions """ submission_encrypted = False - submission_element = ET.fromstring(xml) - encrypted_attrib = submission_element.attrib.get('encrypted') + submission_element = fromstring(xml) + encrypted_attrib = submission_element.attrib.get("encrypted") required_encryption_elems = [ - elem.tag for elem in submission_element - if elem.tag in REQUIRED_ENCRYPTED_FILE_ELEMENTS] + elem.tag + for elem in submission_element + if elem.tag in REQUIRED_ENCRYPTED_FILE_ELEMENTS + ] encryption_elems_num = len(required_encryption_elems) # Check the validity of the submission if encrypted_attrib == "yes" or encryption_elems_num > 1: - if (not encryption_elems_num == 2 or - not encrypted_attrib == "yes") and xform.encrypted: - raise InstanceFormatError( - _("Encrypted submission incorrectly formatted.")) + if ( + not encryption_elems_num == 2 or not encrypted_attrib == "yes" + ) and xform.encrypted: + raise InstanceFormatError(_("Encrypted submission incorrectly formatted.")) submission_encrypted = True if xform.encrypted and not submission_encrypted: - raise InstanceEncryptionError(_( - "Unencrypted submissions are not allowed for encrypted forms.")) + raise InstanceEncryptionError( + _("Unencrypted submissions are not allowed for encrypted forms.") + ) def update_attachment_tracking(instance): @@ -339,8 +398,9 @@ def update_attachment_tracking(instance): instance.total_media = instance.num_of_media instance.media_count = instance.attachments_count instance.media_all_received = instance.media_count == instance.total_media - instance.save(update_fields=['total_media', 'media_count', - 'media_all_received', 'json']) + instance.save( + update_fields=["total_media", "media_count", "media_all_received", "json"] + ) def save_attachments(xform, instance, media_files, remove_deleted_media=False): @@ -351,9 +411,8 @@ def save_attachments(xform, instance, media_files, remove_deleted_media=False): for f in media_files: filename, extension = os.path.splitext(f.name) - extension = extension.replace('.', '') - content_type = u'text/xml' \ - if extension == Attachment.OSM else f.content_type + extension = extension.replace(".", "") + content_type = "text/xml" if extension == Attachment.OSM else f.content_type if extension == Attachment.OSM and not xform.instances_with_osm: xform.instances_with_osm = True xform.save() @@ -361,43 +420,52 @@ def save_attachments(xform, instance, media_files, remove_deleted_media=False): # Validate Attachment file name length if len(filename) > 100: raise AttachmentNameError(filename) - media_in_submission = ( - filename in instance.get_expected_media() or - [instance.xml.decode('utf-8').find(filename) != -1 if - isinstance(instance.xml, bytes) else - instance.xml.find(filename) != -1]) + media_in_submission = filename in instance.get_expected_media() or [ + instance.xml.decode("utf-8").find(filename) != -1 + if isinstance(instance.xml, bytes) + else instance.xml.find(filename) != -1 + ] if media_in_submission: Attachment.objects.get_or_create( instance=instance, media_file=f, mimetype=content_type, name=filename, - extension=extension) + extension=extension, + ) if remove_deleted_media: instance.soft_delete_attachments() update_attachment_tracking(instance) -def save_submission(xform, xml, media_files, new_uuid, submitted_by, status, - date_created_override, checksum, request=None): +def save_submission( + xform, + xml, + media_files, + new_uuid, + submitted_by, + status, + date_created_override, + checksum, + request=None, +): + """Persist a submission into the ParsedInstance model.""" if not date_created_override: date_created_override = get_submission_date_from_xml(xml) - instance = _get_instance(xml, new_uuid, submitted_by, status, xform, - checksum, request) - save_attachments( - xform, - instance, - media_files, - remove_deleted_media=True) + instance = _get_instance( + xml, new_uuid, submitted_by, status, xform, checksum, request + ) + save_attachments(xform, instance, media_files, remove_deleted_media=True) # override date created if required if date_created_override: if not timezone.is_aware(date_created_override): # default to utc? date_created_override = timezone.make_aware( - date_created_override, timezone.utc) + date_created_override, timezone.utc + ) instance.date_created = date_created_override instance.save() @@ -416,13 +484,16 @@ def get_filtered_instances(*args, **kwargs): return Instance.objects.filter(*args, **kwargs) -def create_instance(username, - xml_file, - media_files, - status=u'submitted_via_web', - uuid=None, - date_created_override=None, - request=None): +# pylint: disable=too-many-locals +def create_instance( + username, + xml_file, + media_files, + status="submitted_via_web", + uuid=None, + date_created_override=None, + request=None, +): """ I used to check if this file had been submitted already, I've taken this out because it was too slow. Now we're going to create @@ -433,8 +504,7 @@ def create_instance(username, * If there is a username and a uuid, submitting a new ODK form. """ instance = None - submitted_by = request.user \ - if request and request.user.is_authenticated else None + submitted_by = request.user if request and request.user.is_authenticated else None if username: username = username.lower() @@ -447,18 +517,16 @@ def create_instance(username, new_uuid = get_uuid_from_xml(xml) filtered_instances = get_filtered_instances( - Q(checksum=checksum) | Q(uuid=new_uuid), xform_id=xform.pk) - existing_instance = get_first_record(filtered_instances.only('id')) - if existing_instance and \ - (new_uuid or existing_instance.xform.has_start_time): + Q(checksum=checksum) | Q(uuid=new_uuid), xform_id=xform.pk + ) + existing_instance = get_first_record(filtered_instances.only("id")) + if existing_instance and (new_uuid or existing_instance.xform.has_start_time): # ensure we have saved the extra attachments with transaction.atomic(): save_attachments( - xform, - existing_instance, - media_files, - remove_deleted_media=True) - existing_instance.save(update_fields=['json', 'date_modified']) + xform, existing_instance, media_files, remove_deleted_media=True + ) + existing_instance.save(update_fields=["json", "date_modified"]) # Ignore submission as a duplicate IFF # * a submission's XForm collects start time @@ -467,20 +535,23 @@ def create_instance(username, return DuplicateInstance() # get new and deprecated UUIDs - history = InstanceHistory.objects.filter( - xform_instance__xform_id=xform.pk, - xform_instance__deleted_at__isnull=True, - uuid=new_uuid).only('xform_instance').first() + history = ( + InstanceHistory.objects.filter( + xform_instance__xform_id=xform.pk, + xform_instance__deleted_at__isnull=True, + uuid=new_uuid, + ) + .only("xform_instance") + .first() + ) if history: duplicate_instance = history.xform_instance # ensure we have saved the extra attachments with transaction.atomic(): save_attachments( - xform, - duplicate_instance, - media_files, - remove_deleted_media=True) + xform, duplicate_instance, media_files, remove_deleted_media=True + ) duplicate_instance.save() return DuplicateInstance() @@ -488,23 +559,31 @@ def create_instance(username, try: with transaction.atomic(): if isinstance(xml, bytes): - xml = xml.decode('utf-8') - instance = save_submission(xform, xml, media_files, new_uuid, - submitted_by, status, - date_created_override, checksum, - request) + xml = xml.decode("utf-8") + instance = save_submission( + xform, + xml, + media_files, + new_uuid, + submitted_by, + status, + date_created_override, + checksum, + request, + ) except IntegrityError: - instance = get_first_record(Instance.objects.filter( - Q(checksum=checksum) | Q(uuid=new_uuid), - xform_id=xform.pk)) + instance = get_first_record( + Instance.objects.filter( + Q(checksum=checksum) | Q(uuid=new_uuid), xform_id=xform.pk + ) + ) if instance: attachment_names = [ - a.media_file.name.split('/')[-1] + a.media_file.name.split("/")[-1] for a in Attachment.objects.filter(instance=instance) ] - media_files = [f for f in media_files - if f.name not in attachment_names] + media_files = [f for f in media_files if f.name not in attachment_names] save_attachments(xform, instance, media_files) instance.save() @@ -512,10 +591,16 @@ def create_instance(username, return instance +# pylint: disable=too-many-branches,too-many-statements @use_master -def safe_create_instance( - username, xml_file, media_files, uuid, request, - instance_status: str = 'submitted_via_web'): +def safe_create_instance( # noqa C901 + username, + xml_file, + media_files, + uuid, + request, + instance_status: str = "submitted_via_web", +): """Create an instance and catch exceptions. :returns: A list [error, instance] where error is None if there was no @@ -525,13 +610,19 @@ def safe_create_instance( try: instance = create_instance( - username, xml_file, media_files, uuid=uuid, request=request, - status=instance_status) + username, + xml_file, + media_files, + uuid=uuid, + request=request, + status=instance_status, + ) except InstanceInvalidUserError: - error = OpenRosaResponseBadRequest(_(u"Username or ID required.")) + error = OpenRosaResponseBadRequest(_("Username or ID required.")) except InstanceEmptyError: error = OpenRosaResponseBadRequest( - _(u"Received empty submission. No instance was created")) + _("Received empty submission. No instance was created") + ) except InstanceEncryptionError as e: error = OpenRosaResponseBadRequest(text(e)) except InstanceFormatError as e: @@ -539,100 +630,103 @@ def safe_create_instance( except (FormInactiveError, FormIsMergedDatasetError) as e: error = OpenRosaResponseNotAllowed(text(e)) except XForm.DoesNotExist: - error = OpenRosaResponseNotFound( - _(u"Form does not exist on this account")) + error = OpenRosaResponseNotFound(_("Form does not exist on this account")) except ExpatError: - error = OpenRosaResponseBadRequest(_(u"Improperly formatted XML.")) + error = OpenRosaResponseBadRequest(_("Improperly formatted XML.")) except AttachmentNameError: response = OpenRosaResponseBadRequest( - _("Attachment file name exceeds 100 chars")) + _("Attachment file name exceeds 100 chars") + ) response.status_code = 400 error = response except DuplicateInstance: - response = OpenRosaResponse(_(u"Duplicate submission")) + response = OpenRosaResponse(_("Duplicate submission")) response.status_code = 202 if request: - response['Location'] = request.build_absolute_uri(request.path) + response["Location"] = request.build_absolute_uri(request.path) error = response except PermissionDenied as e: error = OpenRosaResponseForbidden(e) except UnreadablePostError as e: - error = OpenRosaResponseBadRequest( - _(u"Unable to read submitted file: %(error)s" - % {'error': text(e)})) + error = OpenRosaResponseBadRequest(_(f"Unable to read submitted file: {e}")) except InstanceMultipleNodeError as e: error = OpenRosaResponseBadRequest(e) except DjangoUnicodeDecodeError: error = OpenRosaResponseBadRequest( - _(u"File likely corrupted during " - u"transmission, please try later.")) + _("File likely corrupted during " "transmission, please try later.") + ) except NonUniqueFormIdError: error = OpenRosaResponseBadRequest( - _(u"Unable to submit because there are multiple forms with" - u" this formID.")) + _("Unable to submit because there are multiple forms with" " this formID.") + ) except DataError as e: error = OpenRosaResponseBadRequest((str(e))) if isinstance(instance, DuplicateInstance): - response = OpenRosaResponse(_(u"Duplicate submission")) + response = OpenRosaResponse(_("Duplicate submission")) response.status_code = 202 if request: - response['Location'] = request.build_absolute_uri(request.path) + response["Location"] = request.build_absolute_uri(request.path) error = response instance = None return [error, instance] -def response_with_mimetype_and_name(mimetype, - name, - extension=None, - show_date=True, - file_path=None, - use_local_filesystem=False, - full_mime=False): +def response_with_mimetype_and_name( + mimetype, + name, + extension=None, + show_date=True, + file_path=None, + use_local_filesystem=False, + full_mime=False, +): + """Returns a HttpResponse with Content-Disposition header set + + Triggers a download on the browser.""" if extension is None: extension = mimetype if not full_mime: - mimetype = "application/%s" % mimetype + mimetype = f"application/{mimetype}" if file_path: try: if not use_local_filesystem: default_storage = get_storage_class()() wrapper = FileWrapper(default_storage.open(file_path)) - response = StreamingHttpResponse( - wrapper, content_type=mimetype) - response['Content-Length'] = default_storage.size(file_path) + response = StreamingHttpResponse(wrapper, content_type=mimetype) + response["Content-Length"] = default_storage.size(file_path) else: - wrapper = FileWrapper(open(file_path)) - response = StreamingHttpResponse( - wrapper, content_type=mimetype) - response['Content-Length'] = os.path.getsize(file_path) + # pylint: disable=consider-using-with + wrapper = FileWrapper(open(file_path, "rb")) + response = StreamingHttpResponse(wrapper, content_type=mimetype) + response["Content-Length"] = os.path.getsize(file_path) except IOError: - response = HttpResponseNotFound( - _(u"The requested file could not be found.")) + response = HttpResponseNotFound(_("The requested file could not be found.")) else: response = HttpResponse(content_type=mimetype) - response['Content-Disposition'] = generate_content_disposition_header( - name, extension, show_date) + response["Content-Disposition"] = generate_content_disposition_header( + name, extension, show_date + ) return response def generate_content_disposition_header(name, extension, show_date=True): + """Returns the a Content-Description header formatting string,""" if name is None: - return 'attachment;' + return "attachment;" if show_date: - name = "%s-%s" % (name, datetime.now().strftime("%Y-%m-%d-%H-%M-%S")) - return 'attachment; filename=%s.%s' % (name, extension) + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + name = f"{name}-{timestamp}" + return f"attachment; filename={name}.{extension}" def store_temp_file(data): - tmp = tempfile.TemporaryFile() + """Creates a temporary file with the ``data`` and returns it.""" ret = None - try: + with tempfile.TemporaryFile() as tmp: tmp.write(data) tmp.seek(0) ret = tmp - finally: - tmp.close() + return ret @@ -644,37 +738,34 @@ def publish_form(callback): try: return callback() except (PyXFormError, XLSFormError) as e: - return {'type': 'alert-error', 'text': text(e)} + return {"type": "alert-error", "text": text(e)} except IntegrityError: return { - 'type': 'alert-error', - 'text': _(u'Form with this id or SMS-keyword already exists.'), + "type": "alert-error", + "text": _("Form with this id or SMS-keyword already exists."), } except ProcessTimedOut: # catch timeout errors return { - 'type': 'alert-error', - 'text': _(u'Form validation timeout, please try again.'), + "type": "alert-error", + "text": _("Form validation timeout, please try again."), } except (MemoryError, OSError, BadStatusLine): return { - 'type': 'alert-error', - 'text': _((u'An error occurred while publishing the form. ' - 'Please try again.')), + "type": "alert-error", + "text": _( + ("An error occurred while publishing the form. " "Please try again.") + ), } - except (AttributeError, Exception, ValidationError) as e: - report_exception("Form publishing exception: {}".format(e), text(e), - sys.exc_info()) - return {'type': 'alert-error', 'text': text(e)} + except (AttributeError, DuplicateUUIDError, ValidationError) as e: + report_exception(f"Form publishing exception: {e}", text(e), sys.exc_info()) + return {"type": "alert-error", "text": text(e)} @track_object_event( - user_field='user', - properties={ - 'created_by': 'user', - 'xform_id': 'pk', - 'xform_name': 'title'}, - additional_context={'from': 'Publish XLS Form'} + user_field="user", + properties={"created_by": "user", "xform_id": "pk", "xform_name": "title"}, + additional_context={"from": "Publish XLS Form"}, ) @transaction.atomic() def publish_xls_form(xls_file, user, project, id_string=None, created_by=None): @@ -683,16 +774,13 @@ def publish_xls_form(xls_file, user, project, id_string=None, created_by=None): """ # get or create DataDictionary based on user and id string if id_string: - dd = DataDictionary.objects.get( - user=user, id_string=id_string, project=project) + dd = DataDictionary.objects.get(user=user, id_string=id_string, project=project) dd.xls = xls_file dd.save() else: dd = DataDictionary.objects.create( - created_by=created_by or user, - user=user, - xls=xls_file, - project=project) + created_by=created_by or user, user=user, xls=xls_file, project=project + ) # Create an XFormVersion object for the published XLSForm create_xform_version(dd, user) @@ -700,41 +788,35 @@ def publish_xls_form(xls_file, user, project, id_string=None, created_by=None): @track_object_event( - user_field='user', - properties={ - 'created_by': 'user', - 'xform_id': 'pk', - 'xform_name': 'title'}, - additional_context={'from': 'Publish XML Form'} + user_field="user", + properties={"created_by": "user", "xform_id": "pk", "xform_name": "title"}, + additional_context={"from": "Publish XML Form"}, ) def publish_xml_form(xml_file, user, project, id_string=None, created_by=None): + """Publish an XML XForm.""" xml = xml_file.read() if isinstance(xml, bytes): - xml = xml.decode('utf-8') + xml = xml.decode("utf-8") survey = create_survey_element_from_xml(xml) form_json = survey.to_json() if id_string: - dd = DataDictionary.objects.get( - user=user, id_string=id_string, project=project) + dd = DataDictionary.objects.get(user=user, id_string=id_string, project=project) dd.xml = xml dd.json = form_json - dd._mark_start_time_boolean() + dd.mark_start_time_boolean() set_uuid(dd) - dd._set_uuid_in_xml() - dd._set_hash() + dd.set_uuid_in_xml() + dd.set_hash() dd.save() else: created_by = created_by or user dd = DataDictionary( - created_by=created_by, - user=user, - xml=xml, - json=form_json, - project=project) - dd._mark_start_time_boolean() + created_by=created_by, user=user, xml=xml, json=form_json, project=project + ) + dd.mark_start_time_boolean() set_uuid(dd) - dd._set_uuid_in_xml(file_name=xml_file.name) - dd._set_hash() + dd.set_uuid_in_xml(file_name=xml_file.name) + dd.set_hash() dd.save() # Create an XFormVersion object for the published XLSForm @@ -755,60 +837,75 @@ def remove_metadata_fields(data): return data +def set_default_openrosa_headers(response): + """Sets the default OpenRosa headers into a ``response`` object.""" + response["Content-Type"] = "text/html; charset=utf-8" + response["X-OpenRosa-Accept-Content-Length"] = DEFAULT_CONTENT_LENGTH + tz = pytz.timezone(settings.TIME_ZONE) + dt = datetime.now(tz).strftime("%a, %d %b %Y %H:%M:%S %Z") + response["Date"] = dt + response[OPEN_ROSA_VERSION_HEADER] = OPEN_ROSA_VERSION + response["Content-Type"] = DEFAULT_CONTENT_TYPE + + class BaseOpenRosaResponse(HttpResponse): + """The base HTTP response class with OpenRosa headers.""" + status_code = 201 def __init__(self, *args, **kwargs): - super(BaseOpenRosaResponse, self).__init__(*args, **kwargs) - - self[OPEN_ROSA_VERSION_HEADER] = OPEN_ROSA_VERSION - tz = pytz.timezone(settings.TIME_ZONE) - dt = datetime.now(tz).strftime('%a, %d %b %Y %H:%M:%S %Z') - self['Date'] = dt - self['X-OpenRosa-Accept-Content-Length'] = DEFAULT_CONTENT_LENGTH - self['Content-Type'] = DEFAULT_CONTENT_TYPE + super().__init__(*args, **kwargs) + set_default_openrosa_headers(self) class OpenRosaResponse(BaseOpenRosaResponse): + """An HTTP response class with OpenRosa headers for the created response.""" + status_code = 201 def __init__(self, *args, **kwargs): - super(OpenRosaResponse, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.message = self.content # wrap content around xml - self.content = ''' + self.content = f""" - %s -''' % self.content + {self.content} +""" class OpenRosaResponseNotFound(OpenRosaResponse): + """An HTTP response class with OpenRosa headers for the Not Found response.""" + status_code = 404 class OpenRosaResponseBadRequest(OpenRosaResponse): + """An HTTP response class with OpenRosa headers for the Bad Request response.""" + status_code = 400 class OpenRosaResponseNotAllowed(OpenRosaResponse): + """An HTTP response class with OpenRosa headers for the Not Allowed response.""" + status_code = 405 class OpenRosaResponseForbidden(OpenRosaResponse): + """An HTTP response class with OpenRosa headers for the Forbidden response.""" + status_code = 403 class OpenRosaNotAuthenticated(Response): + """An HTTP response class with OpenRosa headers for the Not Authenticated + response.""" + status_code = 401 def __init__(self, *args, **kwargs): - super(OpenRosaNotAuthenticated, self).__init__(*args, **kwargs) - - self['Content-Type'] = 'text/html; charset=utf-8' - self['X-OpenRosa-Accept-Content-Length'] = DEFAULT_CONTENT_LENGTH - tz = pytz.timezone(settings.TIME_ZONE) - dt = datetime.now(tz).strftime('%a, %d %b %Y %H:%M:%S %Z') - self['Date'] = dt + super().__init__(*args, **kwargs) + set_default_openrosa_headers(self) def inject_instanceid(xml_str, uuid): @@ -821,7 +918,8 @@ def inject_instanceid(xml_str, uuid): # check if we have a meta tag survey_node = children.item(0) meta_tags = [ - n for n in survey_node.childNodes + n + for n in survey_node.childNodes if n.nodeType == Node.ELEMENT_NODE and n.tagName.lower() == "meta" ] if len(meta_tags) == 0: @@ -832,7 +930,8 @@ def inject_instanceid(xml_str, uuid): # check if we have an instanceID tag uuid_tags = [ - n for n in meta_tag.childNodes + n + for n in meta_tag.childNodes if n.nodeType == Node.ELEMENT_NODE and n.tagName == "instanceID" ] if len(uuid_tags) == 0: @@ -841,22 +940,26 @@ def inject_instanceid(xml_str, uuid): else: uuid_tag = uuid_tags[0] # insert meta and instanceID - text_node = xml.createTextNode(u"uuid:%s" % uuid) + text_node = xml.createTextNode(f"uuid:{uuid}") uuid_tag.appendChild(text_node) return xml.toxml() return xml_str def remove_xform(xform): + """Deletes an XForm ``xform``.""" # delete xform, and all related models xform.delete() -class PublishXForm(object): +class PublishXForm: + "A class to publish an XML XForm file." + def __init__(self, xml_file, user): self.xml_file = xml_file self.user = user self.project = get_user_default_project(user) def publish_xform(self): + """Publish an XForm XML file.""" return publish_xml_form(self.xml_file, self.user, self.project) diff --git a/onadata/libs/utils/middleware.py b/onadata/libs/utils/middleware.py index d7dfa139aa..25d6eaa7d5 100644 --- a/onadata/libs/utils/middleware.py +++ b/onadata/libs/utils/middleware.py @@ -7,7 +7,7 @@ from django.http import HttpResponseNotAllowed from django.template import loader from django.middleware.locale import LocaleMiddleware -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.utils.translation.trans_real import parse_accept_lang_header from multidb.pinning import use_master @@ -48,7 +48,7 @@ class LocaleMiddlewareWithTweaks(LocaleMiddleware): """ def process_request(self, request): - accept = request.META.get('HTTP_ACCEPT_LANGUAGE', '') + accept = request.headers.get('Accept-Language', '') try: codes = [code for code, r in parse_accept_lang_header(accept)] if 'km' in codes and 'km-kh' not in codes: diff --git a/onadata/libs/utils/openid_connect_tools.py b/onadata/libs/utils/openid_connect_tools.py index cadf33fddf..73c8f48046 100644 --- a/onadata/libs/utils/openid_connect_tools.py +++ b/onadata/libs/utils/openid_connect_tools.py @@ -5,7 +5,7 @@ from django.http import HttpResponseRedirect, Http404 from django.core.cache import cache -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ import jwt import requests diff --git a/onadata/libs/utils/osm.py b/onadata/libs/utils/osm.py index 23393476c1..cfad61280e 100644 --- a/onadata/libs/utils/osm.py +++ b/onadata/libs/utils/osm.py @@ -6,12 +6,12 @@ import logging -from django.contrib.gis.geos import (GeometryCollection, LineString, Point, - Polygon) +from django.contrib.gis.geos import GeometryCollection, LineString, Point, Polygon from django.contrib.gis.geos.error import GEOSException from django.db import IntegrityError, models, transaction -from future.utils import iteritems -from lxml import etree + +from defusedxml.lxml import _etree, fromstring, tostring +from six import iteritems from onadata.apps.logger.models.attachment import Attachment from onadata.apps.logger.models.instance import Instance @@ -24,20 +24,21 @@ def _get_xml_obj(xml): if not isinstance(xml, bytes): xml = xml.strip().encode() try: - return etree.fromstring(xml) # pylint: disable=E1101 - except etree.XMLSyntaxError as e: # pylint: disable=E1101 - if 'Attribute action redefined' in e.msg: - xml = xml.replace(b'action="modify" ', b'') + return fromstring(xml) + except _etree.XMLSyntaxError as e: # pylint: disable=no-member + if "Attribute action redefined" in e.msg: + xml = xml.replace(b'action="modify" ', b"") return _get_xml_obj(xml) + return None def _get_node(ref, root): point = None - nodes = root.xpath('//node[@id="{}"]'.format(ref)) + nodes = root.xpath(f'//node[@id="{ref}"]') if nodes: node = nodes[0] - point = Point(float(node.get('lon')), float(node.get('lat'))) + point = Point(float(node.get("lon")), float(node.get("lat"))) return point @@ -46,9 +47,10 @@ def get_combined_osm(osm_list): """ Combine osm xml form list of OsmData objects """ - xml = '' - if (osm_list and isinstance(osm_list, list)) \ - or isinstance(osm_list, models.QuerySet): + xml = "" + if (osm_list and isinstance(osm_list, list)) or isinstance( + osm_list, models.QuerySet + ): osm = None for osm_data in osm_list: osm_xml = osm_data.xml @@ -61,66 +63,60 @@ def get_combined_osm(osm_list): for child in _osm.getchildren(): osm.append(child) if osm is not None: - # pylint: disable=E1101 - return etree.tostring(osm, encoding='utf-8', xml_declaration=True) - elif isinstance(osm_list, dict): - if 'detail' in osm_list: - xml = '%s' % osm_list['detail'] - return xml.encode('utf-8') + # pylint: disable=no-member + return tostring(osm, encoding="utf-8", xml_declaration=True) + if isinstance(osm_list, dict): + if "detail" in osm_list: + xml = f"{osm_list['detail']}" + return xml.encode("utf-8") def parse_osm_ways(osm_xml, include_osm_id=False): - """Converts an OSM XMl to a list of GEOSGeometry objects """ + """Converts an OSM XMl to a list of GEOSGeometry objects""" items = [] root = _get_xml_obj(osm_xml) - for way in root.findall('way'): + for way in root.findall("way"): geom = None points = [] - for node in way.findall('nd'): - points.append(_get_node(node.get('ref'), root)) + for node in way.findall("nd"): + points.append(_get_node(node.get("ref"), root)) try: geom = Polygon(points) except GEOSException: geom = LineString(points) tags = parse_osm_tags(way, include_osm_id) - items.append({ - 'osm_id': way.get('id'), - 'geom': geom, - 'tags': tags, - 'osm_type': 'way' - }) + items.append( + {"osm_id": way.get("id"), "geom": geom, "tags": tags, "osm_type": "way"} + ) return items def parse_osm_nodes(osm_xml, include_osm_id=False): - """Converts an OSM XMl to a list of GEOSGeometry objects """ + """Converts an OSM XMl to a list of GEOSGeometry objects""" items = [] root = _get_xml_obj(osm_xml) - for node in root.findall('node'): - point = Point(float(node.get('lon')), float(node.get('lat'))) + for node in root.findall("node"): + point = Point(float(node.get("lon")), float(node.get("lat"))) tags = parse_osm_tags(node, include_osm_id) - items.append({ - 'osm_id': node.get('id'), - 'geom': point, - 'tags': tags, - 'osm_type': 'node' - }) + items.append( + {"osm_id": node.get("id"), "geom": point, "tags": tags, "osm_type": "node"} + ) return items def parse_osm_tags(node, include_osm_id=False): """Retrieves all the tags from a osm xml node""" - tags = {} if not include_osm_id else {node.tag + ':id': node.get('id')} - for tag in node.findall('tag'): - key, val = tag.attrib['k'], tag.attrib['v'] - if val == '' or val.upper() == 'FIXME': + tags = {} if not include_osm_id else {node.tag + ":id": node.get("id")} + for tag in node.findall("tag"): + key, val = tag.attrib["k"], tag.attrib["v"] + if val == "" or val.upper() == "FIXME": continue tags.update({key: val}) @@ -153,32 +149,32 @@ def save_osm_data(instance_id): Includes the OSM data in the specified submission json data. """ instance = Instance.objects.filter(pk=instance_id).first() - osm_attachments = instance.attachments.filter(extension=Attachment.OSM) \ - if instance else None + osm_attachments = ( + instance.attachments.filter(extension=Attachment.OSM) if instance else None + ) if instance and osm_attachments: fields = [ f.get_abbreviated_xpath() - for f in instance.xform.get_survey_elements_of_type('osm') + for f in instance.xform.get_survey_elements_of_type("osm") ] osm_filenames = { - field: instance.json[field] - for field in fields if field in instance.json + field: instance.json[field] for field in fields if field in instance.json } for osm in osm_attachments: try: osm_xml = osm.media_file.read() if isinstance(osm_xml, bytes): - osm_xml = osm_xml.decode('utf-8') + osm_xml = osm_xml.decode("utf-8") except IOError as e: - logging.exception("IOError saving osm data: %s" % str(e)) + logging.exception("IOError saving osm data: %s", str(e)) continue else: filename = None field_name = None for k, v in osm_filenames.items(): - if osm.filename.startswith(v.replace('.osm', '')): + if osm.filename.startswith(v.replace(".osm", "")): filename = v field_name = k break @@ -193,26 +189,27 @@ def save_osm_data(instance_id): osm_data = OsmData( instance=instance, xml=osm_xml, - osm_id=osmd['osm_id'], - osm_type=osmd['osm_type'], - tags=osmd['tags'], - geom=GeometryCollection(osmd['geom']), + osm_id=osmd["osm_id"], + osm_type=osmd["osm_type"], + tags=osmd["tags"], + geom=GeometryCollection(osmd["geom"]), filename=filename, - field_name=field_name) + field_name=field_name, + ) osm_data.save() except IntegrityError: with transaction.atomic(): - osm_data = OsmData.objects.exclude( - xml=osm_xml).filter( - instance=instance, - field_name=field_name).first() + osm_data = ( + OsmData.objects.exclude(xml=osm_xml) + .filter(instance=instance, field_name=field_name) + .first() + ) if osm_data: osm_data.xml = osm_xml - osm_data.osm_id = osmd['osm_id'] - osm_data.osm_type = osmd['osm_type'] - osm_data.tags = osmd['tags'] - osm_data.geom = GeometryCollection( - osmd['geom']) + osm_data.osm_id = osmd["osm_id"] + osm_data.osm_type = osmd["osm_type"] + osm_data.tags = osmd["tags"] + osm_data.geom = GeometryCollection(osmd["geom"]) osm_data.filename = filename osm_data.save() instance.save() @@ -231,6 +228,6 @@ def osm_flat_dict(instance_id): for osm in osm_data: for tag in osm.tags: for (k, v) in iteritems(tag): - tags.update({"osm_{}".format(k): v}) + tags.update({f"osm_{k}": v}) return tags diff --git a/onadata/libs/utils/qrcode.py b/onadata/libs/utils/qrcode.py index 7fade1e99c..0f13a4c238 100644 --- a/onadata/libs/utils/qrcode.py +++ b/onadata/libs/utils/qrcode.py @@ -1,28 +1,40 @@ +# -*- coding: utf-8 -*- +""" +QR code utility function. +""" +from io import BytesIO from base64 import b64encode from elaphe import barcode -from io import BytesIO -from past.builtins import basestring -def generate_qrcode(message, stream=None, - eclevel='M', margin=10, - data_mode='8bits', format='PNG', scale=2.5): - """ Generate a QRCode, settings options and output.""" +# pylint: disable=too-many-arguments +def generate_qrcode(message): + """Generate a QRCode, settings options and output.""" + stream = None + eclevel = "M" + margin = 10 + data_mode = "8bits" + image_format = "PNG" + scale = 2.5 if stream is None: stream = BytesIO() - if isinstance(message, basestring): + if isinstance(message, str): message = message.encode() - img = barcode('qrcode', message, - options=dict(version=9, eclevel=eclevel), - margin=margin, data_mode=data_mode, scale=scale) + img = barcode( + "qrcode", + message, + options=dict(version=9, eclevel=eclevel), + margin=margin, + data_mode=data_mode, + scale=scale, + ) - img.save(stream, format) + img.save(stream, image_format) - datauri = "data:image/png;base64,%s"\ - % b64encode(stream.getvalue()).decode("utf-8") + datauri = f"data:image/png;base64,{b64encode(stream.getvalue()).decode('utf-8')}" stream.close() return datauri diff --git a/onadata/libs/utils/quick_converter.py b/onadata/libs/utils/quick_converter.py index 45db46d9fa..8923c3be36 100644 --- a/onadata/libs/utils/quick_converter.py +++ b/onadata/libs/utils/quick_converter.py @@ -1,11 +1,11 @@ from django import forms -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.viewer.models.data_dictionary import DataDictionary class QuickConverter(forms.Form): - xls_file = forms.FileField(label=ugettext_lazy("XLS File")) + xls_file = forms.FileField(label=gettext_lazy("XLS File")) def publish(self, user): if self.is_valid(): diff --git a/onadata/libs/utils/string.py b/onadata/libs/utils/string.py index 13d8f44105..1ece191bc0 100644 --- a/onadata/libs/utils/string.py +++ b/onadata/libs/utils/string.py @@ -1,6 +1,13 @@ -from past.builtins import basestring +# -*- coding: utf-8 -*- +""" +String utility function str2bool - converts yes, true, t, 1 to True +else returns the argument value v. +""" def str2bool(v): - return v.lower() in ( - 'yes', 'true', 't', '1') if isinstance(v, basestring) else v + """ + String utility function str2bool - converts "yes", "true", "t", "1" to True + else returns the argument value v. + """ + return v.lower() in ("yes", "true", "t", "1") if isinstance(v, str) else v diff --git a/onadata/libs/utils/user_auth.py b/onadata/libs/utils/user_auth.py index 167645a955..425ac4fe1d 100644 --- a/onadata/libs/utils/user_auth.py +++ b/onadata/libs/utils/user_auth.py @@ -1,45 +1,54 @@ +# -*- coding=utf-8 -*- +""" +User authentication utility functions. +""" import base64 import re from functools import wraps -from django.contrib.auth import authenticate -from django.contrib.auth.models import User +from django.contrib.auth import authenticate, get_user_model from django.contrib.sites.models import Site -from django.http import HttpResponse, HttpResponseRedirect +from django.http import HttpResponse from django.shortcuts import get_object_or_404 -from rest_framework.authtoken.models import Token + from guardian.shortcuts import assign_perm, get_perms_for_model +from rest_framework.authtoken.models import Token from onadata.apps.api.models import OrganizationProfile, Team, TempToken from onadata.apps.logger.models import MergedXForm, Note, Project, XForm from onadata.apps.main.models import UserProfile from onadata.libs.utils.viewer_tools import get_form +# pylint: disable=invalid-name +User = get_user_model() + class HttpResponseNotAuthorized(HttpResponse): + """HttpResponse that sets basic authentication prompt.""" + status_code = 401 def __init__(self): HttpResponse.__init__(self) - self['WWW-Authenticate'] =\ - 'Basic realm="%s"' % Site.objects.get_current().name + self["WWW-Authenticate"] = f'Basic realm="{Site.objects.get_current().name}"' def check_and_set_user(request, username): + """Returns a User object or a string to redirect.""" if username != request.user.username: - return HttpResponseRedirect("/%s" % username) + return f"/{username}" content_user = None try: content_user = User.objects.get(username=username) except User.DoesNotExist: - return HttpResponseRedirect("/") + return "/" return content_user def set_profile_data(data, content_user): + """Set user profile data.""" # create empty profile if none exists - profile, created = UserProfile.objects\ - .get_or_create(user=content_user) + profile, _created = UserProfile.objects.get_or_create(user=content_user) location = "" if profile.city: location = profile.city @@ -52,76 +61,81 @@ def set_profile_data(data, content_user): user_instances = profile.num_of_submissions home_page = profile.home_page if home_page and re.match("http", home_page) is None: - home_page = "http://%s" % home_page + home_page = f"http://{home_page}" - data.update({ - 'location': location, - 'user_instances': user_instances, - 'home_page': home_page, - 'num_forms': num_forms, - 'forms': forms, - 'profile': profile, - 'content_user': content_user - }) + data.update( + { + "location": location, + "user_instances": user_instances, + "home_page": home_page, + "num_forms": num_forms, + "forms": forms, + "profile": profile, + "content_user": content_user, + } + ) def has_permission(xform, owner, request, shared=False): + """Checks if the ``request.user`` has the necessary permissions to an ``xform``.""" user = request.user - return shared or xform.shared_data or\ - (hasattr(request, 'session') and - request.session.get('public_link') == xform.uuid) or\ - owner == user or\ - user.has_perm('logger.view_xform', xform) or\ - user.has_perm('logger.change_xform', xform) + return ( + shared + or xform.shared_data + or ( + hasattr(request, "session") + and request.session.get("public_link") == xform.uuid + ) + or owner == user + or user.has_perm("logger.view_xform", xform) + or user.has_perm("logger.change_xform", xform) + ) def has_edit_permission(xform, owner, request, shared=False): + """Checks if the ``request.user`` has edit permissions to the ``xform``.""" user = request.user - return (shared and xform.shared_data) or owner == user or\ - user.has_perm('logger.change_xform', xform) + return ( + (shared and xform.shared_data) + or owner == user + or user.has_perm("logger.change_xform", xform) + ) def check_and_set_user_and_form(username, id_string, request): - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } + """Checks and returns an `xform` and `owner` if ``request.user`` has permission.""" + xform_kwargs = {"id_string__iexact": id_string, "user__username__iexact": username} xform = get_form(xform_kwargs) owner = User.objects.get(username=username) - return [xform, owner] if has_permission(xform, owner, request)\ - else [False, False] + return [xform, owner] if has_permission(xform, owner, request) else [False, False] def check_and_set_form_by_id_string(username, id_string, request): - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } + """Checks xform by ``id_string`` and returns an `xform` if ``request.user`` + has permission.""" + xform_kwargs = {"id_string__iexact": id_string, "user__username__iexact": username} xform = get_form(xform_kwargs) - return xform if has_permission(xform, xform.user, request)\ - else False + return xform if has_permission(xform, xform.user, request) else False def check_and_set_form_by_id(pk, request): + """Checks xform by ``pk`` and returns an `xform` if ``request.user`` + has permission.""" xform = get_object_or_404(XForm, pk=pk) - return xform if has_permission(xform, xform.user, request)\ - else False + return xform if has_permission(xform, xform.user, request) else False def get_xform_and_perms(username, id_string, request): - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } + """Returns the `xform` with the matching ``id_string``, and the permissions the + ``request.user`` has.""" + xform_kwargs = {"id_string__iexact": id_string, "user__username__iexact": username} xform = get_form(xform_kwargs) is_owner = xform.user == request.user - can_edit = is_owner or\ - request.user.has_perm('logger.change_xform', xform) - can_view = can_edit or\ - request.user.has_perm('logger.view_xform', xform) + can_edit = is_owner or request.user.has_perm("logger.change_xform", xform) + can_view = can_edit or request.user.has_perm("logger.view_xform", xform) return [xform, is_owner, can_edit, can_view] @@ -132,9 +146,11 @@ def get_xform_users_with_perms(xform): expensive as the one in use with get_users_with_perms """ user_perms = {} - for p in xform.xformuserobjectpermission_set.all().select_related( - 'user', 'permission').only('user', 'permission__codename', - 'content_object_id'): + for p in ( + xform.xformuserobjectpermission_set.all() + .select_related("user", "permission") + .only("user", "permission__codename", "content_object_id") + ): if p.user.username not in user_perms: user_perms[p.user] = [] user_perms[p.user].append(p.permission.codename) @@ -143,72 +159,81 @@ def get_xform_users_with_perms(xform): def helper_auth_helper(request): + """Authenticates a user via basic authentication.""" if request.user and request.user.is_authenticated: return None # source, http://djangosnippets.org/snippets/243/ - if 'HTTP_AUTHORIZATION' in request.META: - auth = request.META['HTTP_AUTHORIZATION'].split() + if "HTTP_AUTHORIZATION" in request.META: + auth = request.headers["Authorization"].split() if len(auth) == 2 and auth[0].lower() == "basic": - uname, passwd = base64.b64decode(auth[1].encode( - 'utf-8')).decode('utf-8').split(':') + uname, passwd = ( + base64.b64decode(auth[1].encode("utf-8")).decode("utf-8").split(":") + ) user = authenticate(username=uname, password=passwd) if user: request.user = user return None - response = HttpResponseNotAuthorized() - return response + + return HttpResponseNotAuthorized() def basic_http_auth(func): + """A basic authentication decorator.""" + @wraps(func) - def inner(request, *args, **kwargs): + def _inner(request, *args, **kwargs): result = helper_auth_helper(request) if result is not None: return result return func(request, *args, **kwargs) - return inner + return _inner def http_auth_string(username, password): - credentials = base64.b64encode(( - '%s:%s' % (username, password)).encode('utf-8') - ).decode('utf-8').strip() - auth_string = 'Basic %s' % credentials + """Return a basic authentication string with username and password.""" + credentials = ( + base64.b64encode(f"{username}:{password}".encode("utf-8")) + .decode("utf-8") + .strip() + ) + auth_string = f"Basic {credentials}" + return auth_string def add_cors_headers(response): - response['Access-Control-Allow-Origin'] = '*' - response['Access-Control-Allow-Methods'] = 'GET' - response['Access-Control-Allow-Headers'] = ('Accept, Origin,' - ' X-Requested-With,' - ' Authorization') - response['Content-Type'] = 'application/json' + """Add CORS headers to the HttpResponse ``response`` instance.""" + response["Access-Control-Allow-Origin"] = "*" + response["Access-Control-Allow-Methods"] = "GET" + response[ + "Access-Control-Allow-Headers" + ] = "Accept, Origin, X-Requested-With, Authorization" + response["Content-Type"] = "application/json" + return response def set_api_permissions_for_user(user): - models = [ - UserProfile, XForm, MergedXForm, Project, Team, OrganizationProfile, - Note - ] + """Sets the permissions to allow a ``user`` to access the APU.""" + models = [UserProfile, XForm, MergedXForm, Project, Team, OrganizationProfile, Note] for model in models: for perm in get_perms_for_model(model): - assign_perm('%s.%s' % (perm.content_type.app_label, perm.codename), - user) + assign_perm(f"{perm.content_type.app_label}.{perm.codename}", user) def get_user_default_project(user): - name = u"{}'s Project".format(user.username) + """Return's the ``user``'s default project, creates it if it does not exist.'""" + name = f"{user.username}'s Project" user_projects = user.project_owner.filter(name=name, organization=user) if user_projects: project = user_projects[0] else: - metadata = {'description': 'Default Project'} + metadata = {"description": "Default Project"} project = Project.objects.create( - name=name, organization=user, created_by=user, metadata=metadata) + name=name, organization=user, created_by=user, metadata=metadata + ) return project @@ -230,7 +255,5 @@ def invalidate_and_regen_tokens(user): access_token = Token.objects.create(user=user).key temp_token = TempToken.objects.create(user=user).key - return { - 'access_token': access_token, - 'temp_token': temp_token - } + + return {"access_token": access_token, "temp_token": temp_token} diff --git a/onadata/libs/utils/viewer_tools.py b/onadata/libs/utils/viewer_tools.py index 62db834666..1249ee8bab 100644 --- a/onadata/libs/utils/viewer_tools.py +++ b/onadata/libs/utils/viewer_tools.py @@ -1,39 +1,33 @@ # -*- coding: utf-8 -*- -"""Util functions for data views.""" +"""Utility functions for data views.""" import json import os -import requests import sys import zipfile -from builtins import open -from future.utils import iteritems from json.decoder import JSONDecodeError -from tempfile import NamedTemporaryFile from typing import Dict -from xml.dom import minidom - -from future.moves.urllib.parse import urljoin +from defusedxml import minidom from django.conf import settings from django.core.files.storage import get_storage_class -from django.core.files.uploadedfile import InMemoryUploadedFile -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ + +import requests +from six.moves.urllib.parse import urljoin from onadata.libs.exceptions import EnketoError from onadata.libs.utils import common_tags from onadata.libs.utils.common_tags import EXPORT_MIMES from onadata.libs.utils.common_tools import report_exception -SLASH = u"/" +SLASH = "/" def image_urls_for_form(xform): """Return image urls of all image attachments of the xform.""" return sum( - [ - image_urls(s) - for s in xform.instances.filter(deleted_at__isnull=True) - ], []) + [image_urls(s) for s in xform.instances.filter(deleted_at__isnull=True)], [] + ) def get_path(path, suffix): @@ -52,7 +46,7 @@ def image_urls(instance): """ default_storage = get_storage_class()() urls = [] - suffix = settings.THUMB_CONF['medium']['suffix'] + suffix = settings.THUMB_CONF["medium"]["suffix"] for attachment in instance.attachments.all(): path = get_path(attachment.media_file.name, suffix) if default_storage.exists(path): @@ -76,14 +70,15 @@ def parse_xform_instance(xml_str): # THIS IS OKAY FOR OUR USE CASE, BUT OTHER USERS SHOULD BEWARE. survey_data = dict(_path_value_pairs(root_node)) if len(list(_all_attributes(root_node))) != 1: - raise AssertionError(_( - u"There should be exactly one attribute in this document.")) - survey_data.update({ - common_tags.XFORM_ID_STRING: - root_node.getAttribute(u"id"), - common_tags.INSTANCE_DOC_NAME: - root_node.nodeName, - }) + raise AssertionError( + _("There should be exactly one attribute in this document.") + ) + survey_data.update( + { + common_tags.XFORM_ID_STRING: root_node.getAttribute("id"), + common_tags.INSTANCE_DOC_NAME: root_node.nodeName, + } + ) return survey_data @@ -105,8 +100,7 @@ def _path_value_pairs(node): if node.childNodes: # there's no data for this leaf node yield _path(node), None - elif len(node.childNodes) == 1 and \ - node.childNodes[0].nodeType == node.TEXT_NODE: + elif len(node.childNodes) == 1 and node.childNodes[0].nodeType == node.TEXT_NODE: # there is data for this leaf node yield _path(node), node.childNodes[0].nodeValue else: @@ -126,21 +120,6 @@ def _all_attributes(node): yield pair -def django_file(path, field_name, content_type): - """Return an InMemoryUploadedFile object for file uploads.""" - # adapted from here: http://groups.google.com/group/django-users/browse_th\ - # read/thread/834f988876ff3c45/ - file_object = open(path, 'rb') - - return InMemoryUploadedFile( - file=file_object, - field_name=field_name, - name=file_object.name, - content_type=content_type, - size=os.path.getsize(path), - charset=None) - - def export_def_from_filename(filename): """Return file extension and mimetype from filename.""" __, ext = os.path.splitext(filename) @@ -156,38 +135,38 @@ def get_client_ip(request): arguments: request -- HttpRequest object. """ - x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') + x_forwarded_for = request.headers.get("X-Forwarded-For") if x_forwarded_for: - return x_forwarded_for.split(',')[0] + return x_forwarded_for.split(",")[0] - return request.META.get('REMOTE_ADDR') + return request.META.get("REMOTE_ADDR") -def get_enketo_urls(form_url, - id_string, - instance_xml=None, - instance_id=None, - return_url=None, - **kwargs) -> Dict[str, str]: +def get_enketo_urls( + form_url, id_string, instance_xml=None, instance_id=None, return_url=None, **kwargs +) -> Dict[str, str]: """Return Enketo URLs.""" - if (not hasattr(settings, 'ENKETO_URL') or - not hasattr(settings, 'ENKETO_API_ALL_SURVEY_LINKS_PATH') or - not hasattr(settings, 'ENKETO_API_TOKEN') or - settings.ENKETO_API_TOKEN == ''): + if ( + not hasattr(settings, "ENKETO_URL") + or not hasattr(settings, "ENKETO_API_ALL_SURVEY_LINKS_PATH") + or not hasattr(settings, "ENKETO_API_TOKEN") + or settings.ENKETO_API_TOKEN == "" + ): return False - url = urljoin( - settings.ENKETO_URL, settings.ENKETO_API_ALL_SURVEY_LINKS_PATH) + url = urljoin(settings.ENKETO_URL, settings.ENKETO_API_ALL_SURVEY_LINKS_PATH) - values = {'form_id': id_string, 'server_url': form_url} + values = {"form_id": id_string, "server_url": form_url} if instance_id is not None and instance_xml is not None: url = urljoin(settings.ENKETO_URL, settings.ENKETO_API_INSTANCE_PATH) - values.update({ - 'instance': instance_xml, - 'instance_id': instance_id, - # convert to unicode string in python3 compatible way - 'return_url': u'%s' % return_url - }) + values.update( + { + "instance": instance_xml, + "instance_id": instance_id, + # convert to unicode string in python3 compatible way + "return_url": f"{return_url}", + } + ) if kwargs: # Kwargs need to take note of xform variable paths i.e. @@ -197,11 +176,15 @@ def get_enketo_urls(form_url, response = requests.post( url, data=values, - auth=(settings.ENKETO_API_TOKEN, ''), - verify=getattr(settings, 'VERIFY_SSL', True)) + auth=(settings.ENKETO_API_TOKEN, ""), + verify=getattr(settings, "VERIFY_SSL", True), + ) resp_content = response.content - resp_content = resp_content.decode('utf-8') if hasattr( - resp_content, 'decode') else resp_content + resp_content = ( + resp_content.decode("utf-8") + if hasattr(resp_content, "decode") + else resp_content + ) if response.status_code in [200, 201]: try: data = json.loads(resp_content) @@ -213,22 +196,25 @@ def get_enketo_urls(form_url, handle_enketo_error(response) + return None + def handle_enketo_error(response): """Handle enketo error response.""" try: data = json.loads(response.content) - except (ValueError, JSONDecodeError): - report_exception("HTTP Error {}".format(response.status_code), - response.text, sys.exc_info()) + except (ValueError, JSONDecodeError) as e: + report_exception( + f"HTTP Error {response.status_code}", response.text, sys.exc_info() + ) if response.status_code == 502: raise EnketoError( - u"Sorry, we cannot load your form right now. Please try " - "again later.") - raise EnketoError() + "Sorry, we cannot load your form right now. Please try again later." + ) from e + raise EnketoError() from e else: - if 'message' in data: - raise EnketoError(data['message']) + if "message" in data: + raise EnketoError(data["message"]) raise EnketoError(response.text) @@ -237,19 +223,21 @@ def generate_enketo_form_defaults(xform, **kwargs): defaults = {} if kwargs: - for (name, value) in iteritems(kwargs): + for (name, value) in kwargs.items(): field = xform.get_survey_element(name) if field: - defaults["defaults[{}]".format(field.get_xpath())] = value + defaults[f"defaults[{field.get_xpath()}]"] = value return defaults -def create_attachments_zipfile(attachments): - """Return a zip file with submission attachments.""" - # create zip_file - tmp = NamedTemporaryFile() - with zipfile.ZipFile(tmp, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) as z: +def create_attachments_zipfile(attachments, zip_file): + """Return a zip file with submission attachments. + + :param attachments: an Attachments queryset. + :param zip_file: a file object, more likely a NamedTemporaryFile() object. + """ + with zipfile.ZipFile(zip_file, "w", zipfile.ZIP_DEFLATED, allowZip64=True) as z: for attachment in attachments: default_storage = get_storage_class()() filename = attachment.media_file.name @@ -260,29 +248,30 @@ def create_attachments_zipfile(attachments): if f.size > settings.ZIP_REPORT_ATTACHMENT_LIMIT: report_exception( "Create attachment zip exception", - "File is greater than {} bytes".format( - settings.ZIP_REPORT_ATTACHMENT_LIMIT) + ( + "File is greater than " + f"{settings.ZIP_REPORT_ATTACHMENT_LIMIT} bytes" + ), ) break - else: - z.writestr(attachment.media_file.name, f.read()) + z.writestr(attachment.media_file.name, f.read()) except IOError as e: report_exception("Create attachment zip exception", e) break - return tmp - def get_form(kwargs): """Return XForm object by applying kwargs on an XForm queryset.""" # adding inline imports here because adding them at the top of the file # triggers the following error: # django.core.exceptions.AppRegistryNotReady: Apps aren't loaded yet. - from onadata.apps.logger.models import XForm + # pylint: disable=import-outside-toplevel from django.http import Http404 - queryset = kwargs.pop('queryset', XForm.objects.all()) - kwargs['deleted_at__isnull'] = True + from onadata.apps.logger.models import XForm + + queryset = kwargs.pop("queryset", XForm.objects.all()) + kwargs["deleted_at__isnull"] = True xform = queryset.filter(**kwargs).first() if xform: return xform @@ -290,12 +279,15 @@ def get_form(kwargs): raise Http404("XForm does not exist.") -def get_form_url(request, - username=None, - protocol='https', - preview=False, - xform_pk=None, - generate_consistent_urls=False): +# pylint: disable=too-many-arguments +def get_form_url( + request, + username=None, + protocol="https", + preview=False, + xform_pk=None, + generate_consistent_urls=False, +): """ Return a form list url endpoint to be used to make a request to Enketo. @@ -309,17 +301,16 @@ def get_form_url(request, http_host = settings.TEST_HTTP_HOST username = settings.TEST_USERNAME else: - http_host = request.META.get('HTTP_HOST', 'ona.io') + http_host = request.headers.get("Host", "ona.io") - url = '%s://%s' % (protocol, http_host) + url = f"{protocol}://{http_host}" if preview: - url += '/preview' + url += "/preview" if xform_pk and generate_consistent_urls: - url += "/enketo/{}".format(xform_pk) + url += f"/enketo/{xform_pk}" elif username: - url += "/{}/{}".format(username, xform_pk) if xform_pk \ - else "/{}".format(username) + url += f"/{username}/{xform_pk}" if xform_pk else f"/{username}" return url diff --git a/onadata/settings/common.py b/onadata/settings/common.py index fb7945a27c..b4c5892189 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/common.py @@ -1,4 +1,7 @@ # vim: set fileencoding=utf-8 +""" +Base Django settings module. +""" # this system uses structured settings as defined in # http://www.slideshare.net/jacobian/the-best-and-worst-of-django # @@ -12,26 +15,23 @@ import logging import os import socket -import subprocess # noqa, used by included files import sys -from imp import reload +from importlib import reload from celery.signals import after_setup_logger from django.core.exceptions import SuspiciousOperation from django.utils.log import AdminEmailHandler -from past.builtins import basestring # setting default encoding to utf-8 -if sys.version[0] == '2': +if sys.version[0] == "2": reload(sys) sys.setdefaultencoding("utf-8") CURRENT_FILE = os.path.abspath(__file__) -PROJECT_ROOT = os.path.realpath( - os.path.join(os.path.dirname(CURRENT_FILE), '../')) +PROJECT_ROOT = os.path.realpath(os.path.join(os.path.dirname(CURRENT_FILE), "../")) PRINT_EXCEPTION = False -TEMPLATED_EMAIL_TEMPLATE_DIR = 'templated_email/' +TEMPLATED_EMAIL_TEMPLATE_DIR = "templated_email/" ADMINS = ( # ('Your Name', 'your_email@example.com'), @@ -39,9 +39,9 @@ MANAGERS = ADMINS -DEFAULT_FROM_EMAIL = 'noreply@ona.io' -SHARE_PROJECT_SUBJECT = '{} Ona Project has been shared with you.' -SHARE_ORG_SUBJECT = '{}, You have been added to {} organisation.' +DEFAULT_FROM_EMAIL = "noreply@ona.io" +SHARE_PROJECT_SUBJECT = "{} Ona Project has been shared with you." +SHARE_ORG_SUBJECT = "{}, You have been added to {} organisation." DEFAULT_SESSION_EXPIRY_TIME = 21600 # 6 hours DEFAULT_TEMP_TOKEN_EXPIRY_TIME = 21600 # 6 hours @@ -52,21 +52,21 @@ # timezone as the operating system. # If running in a Windows environment this must be set to the same as your # system time zone. -TIME_ZONE = 'America/New_York' +TIME_ZONE = "America/New_York" # Language code for this installation. All choices can be found here: # http://www.i18nguy.com/unicode/language-identifiers.html -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = "en-us" LANGUAGES = ( - ('fr', u'Français'), - ('en', u'English'), - ('es', u'Español'), - ('it', u'Italiano'), - ('km', u'ភាសាខ្មែរ'), - ('ne', u'नेपाली'), - ('nl', u'Nederlands'), - ('zh', u'中文'), + ("fr", "Français"), + ("en", "English"), + ("es", "Español"), + ("it", "Italiano"), + ("km", "ភាសាខ្មែរ"), + ("ne", "नेपाली"), + ("nl", "Nederlands"), + ("zh", "中文"), ) SITE_ID = 1 @@ -82,39 +82,39 @@ # URL that handles the media served from MEDIA_ROOT. Make sure to use a # trailing slash. # Examples: "http://media.lawrence.com/media/", "http://example.com/media/" -MEDIA_URL = 'http://localhost:8000/media/' +MEDIA_URL = "http://localhost:8000/media/" # Absolute path to the directory static files should be collected to. # Don't put anything in this directory yourself; store your static files # in apps' "static/" subdirectories and in STATICFILES_DIRS. # Example: "/home/media/media.lawrence.com/static/" -STATIC_ROOT = os.path.join(PROJECT_ROOT, 'static') +STATIC_ROOT = os.path.join(PROJECT_ROOT, "static") # URL prefix for static files. # Example: "http://media.lawrence.com/static/" -STATIC_URL = '/static/' +STATIC_URL = "/static/" # Enketo URL -ENKETO_PROTOCOL = 'https' -ENKETO_URL = 'https://enketo.ona.io/' -ENKETO_API_ALL_SURVEY_LINKS_PATH = '/api_v2/survey/all' -ENKETO_API_INSTANCE_PATH = '/api_v2/instance' -ENKETO_API_TOKEN = '' +ENKETO_PROTOCOL = "https" +ENKETO_URL = "https://enketo.ona.io/" +ENKETO_API_ALL_SURVEY_LINKS_PATH = "/api_v2/survey/all" +ENKETO_API_INSTANCE_PATH = "/api_v2/instance" +ENKETO_API_TOKEN = "" ENKETO_API_INSTANCE_IFRAME_URL = ENKETO_URL + "api_v2/instance/iframe" -ENKETO_API_SALT = 'secretsalt' +ENKETO_API_SALT = "secretsalt" VERIFY_SSL = True -ENKETO_AUTH_COOKIE = '__enketo' -ENKETO_META_UID_COOKIE = '__enketo_meta_uid' -ENKETO_META_USERNAME_COOKIE = '__enketo_meta_username' +ENKETO_AUTH_COOKIE = "__enketo" +ENKETO_META_UID_COOKIE = "__enketo_meta_uid" +ENKETO_META_USERNAME_COOKIE = "__enketo_meta_username" # Login URLs -LOGIN_URL = '/accounts/login/' -LOGIN_REDIRECT_URL = '/login_redirect/' +LOGIN_URL = "/accounts/login/" +LOGIN_REDIRECT_URL = "/login_redirect/" # URL prefix for admin static files -- CSS, JavaScript and images. # Make sure to use a trailing slash. # Examples: "http://foo.com/static/admin/", "/static/admin/". -ADMIN_MEDIA_PREFIX = '/static/admin/' +ADMIN_MEDIA_PREFIX = "/static/admin/" # Additional locations of static files STATICFILES_DIRS = ( @@ -126,110 +126,113 @@ # List of finder classes that know how to find static files in # various locations. STATICFILES_FINDERS = ( - 'django.contrib.staticfiles.finders.FileSystemFinder', - 'django.contrib.staticfiles.finders.AppDirectoriesFinder', + "django.contrib.staticfiles.finders.FileSystemFinder", + "django.contrib.staticfiles.finders.AppDirectoriesFinder", # 'django.contrib.staticfiles.finders.DefaultStorageFinder', ) TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [ - os.path.join(PROJECT_ROOT, 'libs/templates'), + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [ + os.path.join(PROJECT_ROOT, "libs/templates"), ], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.contrib.auth.context_processors.auth', - 'django.template.context_processors.debug', - 'django.template.context_processors.i18n', - 'django.template.context_processors.media', - 'django.template.context_processors.static', - 'django.template.context_processors.tz', - 'django.contrib.messages.context_processors.messages', - 'onadata.apps.main.context_processors.google_analytics', - 'onadata.apps.main.context_processors.site_name', + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.contrib.auth.context_processors.auth", + "django.template.context_processors.debug", + "django.template.context_processors.i18n", + "django.template.context_processors.media", + "django.template.context_processors.static", + "django.template.context_processors.tz", + "django.template.context_processors.request", + "django.contrib.messages.context_processors.messages", + "onadata.apps.main.context_processors.google_analytics", + "onadata.apps.main.context_processors.site_name", ], }, }, ] +DEFAULT_AUTO_FIELD = "django.db.models.AutoField" MIDDLEWARE = ( - 'onadata.libs.profiling.sql.SqlTimingMiddleware', - 'django.middleware.http.ConditionalGetMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', + "onadata.libs.profiling.sql.SqlTimingMiddleware", + "django.middleware.http.ConditionalGetMiddleware", + "django.middleware.common.CommonMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", # 'django.middleware.locale.LocaleMiddleware', - 'onadata.libs.utils.middleware.LocaleMiddlewareWithTweaks', - 'django.middleware.csrf.CsrfViewMiddleware', - 'corsheaders.middleware.CorsMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'onadata.libs.utils.middleware.HTTPResponseNotAllowedMiddleware', - 'onadata.libs.utils.middleware.OperationalErrorMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', + "onadata.libs.utils.middleware.LocaleMiddlewareWithTweaks", + "django.middleware.csrf.CsrfViewMiddleware", + "corsheaders.middleware.CorsMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "onadata.libs.utils.middleware.HTTPResponseNotAllowedMiddleware", + "onadata.libs.utils.middleware.OperationalErrorMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", ) -X_FRAME_OPTIONS = 'DENY' +X_FRAME_OPTIONS = "DENY" -LOCALE_PATHS = (os.path.join(PROJECT_ROOT, 'onadata.apps.main', 'locale'), ) +LOCALE_PATHS = (os.path.join(PROJECT_ROOT, "onadata.apps.main", "locale"),) -ROOT_URLCONF = 'onadata.apps.main.urls' +ROOT_URLCONF = "onadata.apps.main.urls" USE_TZ = True # needed by guardian -ANONYMOUS_DEFAULT_USERNAME = 'AnonymousUser' +ANONYMOUS_DEFAULT_USERNAME = "AnonymousUser" INSTALLED_APPS = ( - 'django.contrib.contenttypes', - 'django.contrib.auth', - 'django.contrib.sessions', - 'django.contrib.sites', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'django.contrib.humanize', - 'django.contrib.admin', - 'django.contrib.admindocs', - 'django.contrib.gis', - 'registration', - 'django_nose', - 'django_digest', - 'corsheaders', - 'oauth2_provider', - 'rest_framework', - 'rest_framework.authtoken', - 'taggit', - 'onadata.apps.logger', - 'onadata.apps.viewer', - 'onadata.apps.main', - 'onadata.apps.restservice', - 'onadata.apps.api', - 'guardian', - 'onadata.apps.sms_support', - 'onadata.libs', - 'reversion', - 'actstream', - 'onadata.apps.messaging.apps.MessagingConfig', - 'django_filters', - 'oidc', + "django.contrib.contenttypes", + "django.contrib.auth", + "django.contrib.sessions", + "django.contrib.sites", + "django.contrib.messages", + "django.contrib.staticfiles", + "django.contrib.humanize", + "django.contrib.admin", + "django.contrib.admindocs", + "django.contrib.gis", + "registration", + # "django_nose", + "django_digest", + "corsheaders", + "oauth2_provider", + "rest_framework", + "rest_framework.authtoken", + "taggit", + "onadata.apps.logger", + "onadata.apps.viewer", + "onadata.apps.main", + "onadata.apps.restservice", + "onadata.apps.api", + "guardian", + "onadata.apps.sms_support", + "onadata.libs", + "reversion", + "actstream", + "onadata.apps.messaging.apps.MessagingConfig", + "django_filters", + "oidc", ) OAUTH2_PROVIDER = { # this is the list of available scopes - 'SCOPES': { - 'read': 'Read scope', - 'write': 'Write scope', - 'groups': 'Access to your groups'}, - 'OAUTH2_VALIDATOR_CLASS': 'onadata.libs.authentication.MasterReplicaOAuth2Validator' # noqa + "SCOPES": { + "read": "Read scope", + "write": "Write scope", + "groups": "Access to your groups", + }, + "OAUTH2_VALIDATOR_CLASS": "onadata.libs.authentication.MasterReplicaOAuth2Validator", # noqa } OPENID_CONNECT_VIEWSET_CONFIG = { "REDIRECT_AFTER_AUTH": "http://localhost:3000", "USE_SSO_COOKIE": True, "SSO_COOKIE_DATA": "email", - "JWT_SECRET_KEY": 'thesecretkey', - "JWT_ALGORITHM": 'HS256', + "JWT_SECRET_KEY": "thesecretkey", + "JWT_ALGORITHM": "HS256", "SSO_COOKIE_MAX_AGE": None, "SSO_COOKIE_DOMAIN": "localhost", "USE_AUTH_BACKEND": False, @@ -239,80 +242,71 @@ OPENID_CONNECT_AUTH_SERVERS = { "microsoft": { - "AUTHORIZATION_ENDPOINT": - "https://login.microsoftonline.com/common/oauth2/v2.0/authorize", + "AUTHORIZATION_ENDPOINT": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize", "CLIENT_ID": "client_id", - "JWKS_ENDPOINT": - "https://login.microsoftonline.com/common/discovery/v2.0/keys", + "JWKS_ENDPOINT": "https://login.microsoftonline.com/common/discovery/v2.0/keys", "SCOPE": "openid profile", - "TOKEN_ENDPOINT": - "https://login.microsoftonline.com/common/oauth2/v2.0/token", + "TOKEN_ENDPOINT": "https://login.microsoftonline.com/common/oauth2/v2.0/token", "END_SESSION_ENDPOINT": "http://localhost:3000", "REDIRECT_URI": "http://localhost:8000/oidc/msft/callback", "RESPONSE_TYPE": "id_token", "RESPONSE_MODE": "form_post", - "USE_NONCES": True + "USE_NONCES": True, } } REST_FRAMEWORK = { # Use hyperlinked styles by default. # Only used if the `serializer_class` attribute is not set on a view. - 'DEFAULT_MODEL_SERIALIZER_CLASS': - 'rest_framework.serializers.HyperlinkedModelSerializer', - + "DEFAULT_MODEL_SERIALIZER_CLASS": "rest_framework.serializers.HyperlinkedModelSerializer", # Use Django's standard `django.contrib.auth` permissions, # or allow read-only access for unauthenticated users. - 'DEFAULT_PERMISSION_CLASSES': [ - 'rest_framework.permissions.AllowAny', + "DEFAULT_PERMISSION_CLASSES": [ + "rest_framework.permissions.AllowAny", ], - 'DEFAULT_AUTHENTICATION_CLASSES': ( - 'onadata.libs.authentication.DigestAuthentication', - 'onadata.libs.authentication.TempTokenAuthentication', - 'onadata.libs.authentication.EnketoTokenAuthentication', - 'oauth2_provider.contrib.rest_framework.OAuth2Authentication', - 'rest_framework.authentication.SessionAuthentication', - 'rest_framework.authentication.TokenAuthentication', + "DEFAULT_AUTHENTICATION_CLASSES": ( + "onadata.libs.authentication.DigestAuthentication", + "onadata.libs.authentication.TempTokenAuthentication", + "onadata.libs.authentication.EnketoTokenAuthentication", + "oauth2_provider.contrib.rest_framework.OAuth2Authentication", + "rest_framework.authentication.SessionAuthentication", + "rest_framework.authentication.TokenAuthentication", ), - 'DEFAULT_RENDERER_CLASSES': ( - 'rest_framework.renderers.JSONRenderer', - 'rest_framework_jsonp.renderers.JSONPRenderer', - 'rest_framework_csv.renderers.CSVRenderer', + "DEFAULT_RENDERER_CLASSES": ( + "rest_framework.renderers.JSONRenderer", + "rest_framework_jsonp.renderers.JSONPRenderer", + "rest_framework_csv.renderers.CSVRenderer", ), } SWAGGER_SETTINGS = { - "exclude_namespaces": [], # List URL namespaces to ignore - "api_version": '1.0', # Specify your API's version (optional) - "enabled_methods": [ # Methods to enable in UI - 'get', - 'post', - 'put', - 'patch', - 'delete' + "exclude_namespaces": [], # List URL namespaces to ignore + "api_version": "1.0", # Specify your API's version (optional) + "enabled_methods": [ # Methods to enable in UI + "get", + "post", + "put", + "patch", + "delete", ], } CORS_ORIGIN_ALLOW_ALL = False CORS_ALLOW_CREDENTIALS = True -CORS_ORIGIN_WHITELIST = ( - 'http://dev.ona.io', -) -CORS_URLS_ALLOW_ALL_REGEX = ( - r'^/api/v1/osm/.*$', -) +CORS_ORIGIN_WHITELIST = ("http://dev.ona.io",) +CORS_URLS_ALLOW_ALL_REGEX = (r"^/api/v1/osm/.*$",) USE_THOUSAND_SEPARATOR = True COMPRESS = True # extra data stored with users -AUTH_PROFILE_MODULE = 'onadata.apps.main.UserProfile' +AUTH_PROFILE_MODULE = "onadata.apps.main.UserProfile" # case insensitive usernames AUTHENTICATION_BACKENDS = ( - 'onadata.apps.main.backends.ModelBackend', - 'guardian.backends.ObjectPermissionBackend', + "onadata.apps.main.backends.ModelBackend", + "guardian.backends.ObjectPermissionBackend", ) # Settings for Django Registration @@ -343,55 +337,49 @@ def skip_suspicious_operations(record): # See http://docs.djangoproject.com/en/dev/topics/logging for # more details on how to customize your logging configuration. LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'verbose': { - 'format': '%(levelname)s %(asctime)s %(module)s' + - ' %(process)d %(thread)d %(message)s' - }, - 'simple': { - 'format': '%(levelname)s %(message)s' + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "verbose": { + "format": "%(levelname)s %(asctime)s %(module)s" + + " %(process)d %(thread)d %(message)s" }, - 'profiler': { - 'format': '%(levelname)s %(asctime)s %(message)s' + "simple": {"format": "%(levelname)s %(message)s"}, + "profiler": {"format": "%(levelname)s %(asctime)s %(message)s"}, + "sql": { + "format": "%(levelname)s %(process)d %(thread)d" + + " %(time)s seconds %(message)s %(sql)s" }, - 'sql': { - 'format': '%(levelname)s %(process)d %(thread)d' + - ' %(time)s seconds %(message)s %(sql)s' + "sql_totals": { + "format": "%(levelname)s %(process)d %(thread)d %(time)s seconds" + + " %(message)s %(num_queries)s sql queries" }, - 'sql_totals': { - 'format': '%(levelname)s %(process)d %(thread)d %(time)s seconds' + - ' %(message)s %(num_queries)s sql queries' - } }, - 'filters': { - 'require_debug_false': { - '()': 'django.utils.log.RequireDebugFalse' - }, + "filters": { + "require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}, # Define filter for suspicious urls - 'skip_suspicious_operations': { - '()': 'django.utils.log.CallbackFilter', - 'callback': skip_suspicious_operations, + "skip_suspicious_operations": { + "()": "django.utils.log.CallbackFilter", + "callback": skip_suspicious_operations, }, }, - 'handlers': { - 'mail_admins': { - 'level': 'ERROR', - 'filters': ['require_debug_false', 'skip_suspicious_operations'], - 'class': 'django.utils.log.AdminEmailHandler' + "handlers": { + "mail_admins": { + "level": "ERROR", + "filters": ["require_debug_false", "skip_suspicious_operations"], + "class": "django.utils.log.AdminEmailHandler", }, - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'verbose', - 'stream': sys.stdout + "console": { + "level": "DEBUG", + "class": "logging.StreamHandler", + "formatter": "verbose", + "stream": sys.stdout, }, - 'audit': { - 'level': 'DEBUG', - 'class': 'onadata.libs.utils.log.AuditLogHandler', - 'formatter': 'verbose', - 'model': 'onadata.apps.main.models.audit.AuditLog' + "audit": { + "level": "DEBUG", + "class": "onadata.libs.utils.log.AuditLogHandler", + "formatter": "verbose", + "model": "onadata.apps.main.models.audit.AuditLog", }, # 'sql_handler': { # 'level': 'DEBUG', @@ -406,22 +394,18 @@ def skip_suspicious_operations(record): # 'stream': sys.stdout # } }, - 'loggers': { - 'django.request': { - 'handlers': ['mail_admins', 'console'], - 'level': 'DEBUG', - 'propagate': True, - }, - 'console_logger': { - 'handlers': ['console'], - 'level': 'DEBUG', - 'propagate': True + "loggers": { + "django.request": { + "handlers": ["mail_admins", "console"], + "level": "DEBUG", + "propagate": True, }, - 'audit_logger': { - 'handlers': ['audit'], - 'level': 'DEBUG', - 'propagate': True + "console_logger": { + "handlers": ["console"], + "level": "DEBUG", + "propagate": True, }, + "audit_logger": {"handlers": ["audit"], "level": "DEBUG", "propagate": True}, # 'sql_logger': { # 'handlers': ['sql_handler'], # 'level': 'DEBUG', @@ -432,15 +416,18 @@ def skip_suspicious_operations(record): # 'level': 'DEBUG', # 'propagate': True # } - } + }, } # PROFILE_API_ACTION_FUNCTION is used to toggle profiling a viewset's action PROFILE_API_ACTION_FUNCTION = False -PROFILE_LOG_BASE = '/tmp/' +PROFILE_LOG_BASE = "/tmp/" def configure_logging(logger, **kwargs): + """ + Add AdminEmailHandler to the logger + """ admin_email_handler = AdminEmailHandler() admin_email_handler.setLevel(logging.ERROR) logger.addHandler(admin_email_handler) @@ -448,24 +435,24 @@ def configure_logging(logger, **kwargs): after_setup_logger.connect(configure_logging) -GOOGLE_STEP2_URI = 'http://ona.io/gwelcome' -GOOGLE_OAUTH2_CLIENT_ID = 'REPLACE ME' -GOOGLE_OAUTH2_CLIENT_SECRET = 'REPLACE ME' +GOOGLE_STEP2_URI = "http://ona.io/gwelcome" +GOOGLE_OAUTH2_CLIENT_ID = "REPLACE ME" +GOOGLE_OAUTH2_CLIENT_SECRET = "REPLACE ME" THUMB_CONF = { - 'large': {'size': 1280, 'suffix': '-large'}, - 'medium': {'size': 640, 'suffix': '-medium'}, - 'small': {'size': 240, 'suffix': '-small'}, + "large": {"size": 1280, "suffix": "-large"}, + "medium": {"size": 640, "suffix": "-medium"}, + "small": {"size": 240, "suffix": "-small"}, } # order of thumbnails from largest to smallest -THUMB_ORDER = ['large', 'medium', 'small'] -DEFAULT_IMG_FILE_TYPE = 'jpg' +THUMB_ORDER = ["large", "medium", "small"] +DEFAULT_IMG_FILE_TYPE = "jpg" # celery CELERY_TASK_ALWAYS_EAGER = False CELERY_TASK_IGNORE_RESULT = False CELERY_TASK_TRACK_STARTED = True -CELERY_IMPORTS = ('onadata.libs.utils.csv_import',) +CELERY_IMPORTS = ("onadata.libs.utils.csv_import",) CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD = 100000 # Bytes @@ -482,12 +469,12 @@ def configure_logging(logger, **kwargs): # default content length for submission requests DEFAULT_CONTENT_LENGTH = 10000000 -TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' -NOSE_ARGS = ['--with-fixture-bundling', '--nologcapture', '--nocapture'] +# TEST_RUNNER = "django_nose.NoseTestSuiteRunner" +# NOSE_ARGS = ["--with-fixture-bundling", "--nologcapture", "--nocapture"] # fake endpoints for testing -TEST_HTTP_HOST = 'testserver.com' -TEST_USERNAME = 'bob' +TEST_HTTP_HOST = "testserver.com" +TEST_USERNAME = "bob" # specify the root folder which may contain a templates folder and a static # folder used to override templates for site specific details @@ -497,47 +484,47 @@ def configure_logging(logger, **kwargs): BINARY_SELECT_MULTIPLES = False # Use 'n/a' for empty values by default on csv exports -NA_REP = 'n/a' +NA_REP = "n/a" -if isinstance(TEMPLATE_OVERRIDE_ROOT_DIR, basestring): +if isinstance(TEMPLATE_OVERRIDE_ROOT_DIR, str): # site templates overrides - TEMPLATES[0]['DIRS'] = [ - os.path.join(PROJECT_ROOT, TEMPLATE_OVERRIDE_ROOT_DIR, 'templates'), - ] + TEMPLATES[0]['DIRS'] + TEMPLATES[0]["DIRS"] = [ + os.path.join(PROJECT_ROOT, TEMPLATE_OVERRIDE_ROOT_DIR, "templates"), + ] + TEMPLATES[0]["DIRS"] # site static files path STATICFILES_DIRS += ( - os.path.join(PROJECT_ROOT, TEMPLATE_OVERRIDE_ROOT_DIR, 'static'), + os.path.join(PROJECT_ROOT, TEMPLATE_OVERRIDE_ROOT_DIR, "static"), ) # Set wsgi url scheme to HTTPS -os.environ['wsgi.url_scheme'] = 'https' +os.environ["wsgi.url_scheme"] = "https" SUPPORTED_MEDIA_UPLOAD_TYPES = [ - 'audio/mp3', - 'audio/mpeg', - 'audio/wav', - 'audio/x-m4a', - 'image/jpeg', - 'image/png', - 'image/svg+xml', - 'text/csv', - 'text/json', - 'video/3gpp', - 'video/mp4', - 'application/json', - 'application/geo+json', - 'application/pdf', - 'application/msword', - 'application/vnd.ms-excel', - 'application/vnd.ms-powerpoint', - 'application/vnd.oasis.opendocument.text', - 'application/vnd.oasis.opendocument.spreadsheet', - 'application/vnd.oasis.opendocument.presentation', - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - 'application/vnd.openxmlformats-officedocument.presentationml.\ - presentation', - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - 'application/zip', + "audio/mp3", + "audio/mpeg", + "audio/wav", + "audio/x-m4a", + "image/jpeg", + "image/png", + "image/svg+xml", + "text/csv", + "text/json", + "video/3gpp", + "video/mp4", + "application/json", + "application/geo+json", + "application/pdf", + "application/msword", + "application/vnd.ms-excel", + "application/vnd.ms-powerpoint", + "application/vnd.oasis.opendocument.text", + "application/vnd.oasis.opendocument.spreadsheet", + "application/vnd.oasis.opendocument.presentation", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.openxmlformats-officedocument.presentationml.\ + presentation", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/zip", ] CSV_ROW_IMPORT_ASYNC_THRESHOLD = 100 @@ -547,27 +534,26 @@ def configure_logging(logger, **kwargs): PARSED_INSTANCE_DEFAULT_LIMIT = 1000000 PARSED_INSTANCE_DEFAULT_BATCHSIZE = 1000 -PROFILE_SERIALIZER = \ +PROFILE_SERIALIZER = ( "onadata.libs.serializers.user_profile_serializer.UserProfileSerializer" -ORG_PROFILE_SERIALIZER = \ +) +ORG_PROFILE_SERIALIZER = ( "onadata.libs.serializers.organization_serializer.OrganizationSerializer" +) BASE_VIEWSET = "onadata.libs.baseviewset.DefaultBaseViewset" path = os.path.join(PROJECT_ROOT, "..", "extras", "reserved_accounts.txt") EXPORT_WITH_IMAGE_DEFAULT = True try: - with open(path, 'r') as f: + with open(path, "r", encoding="utf-8") as f: RESERVED_USERNAMES = [line.rstrip() for line in f] except EnvironmentError: RESERVED_USERNAMES = [] -STATIC_DOC = '/static/docs/index.html' +STATIC_DOC = "/static/docs/index.html" -try: - HOSTNAME = socket.gethostname() -except Exception: - HOSTNAME = 'localhost' +HOSTNAME = socket.gethostname() CACHE_MIXIN_SECONDS = 60 @@ -578,26 +564,19 @@ def configure_logging(logger, **kwargs): DEFAULT_CELERY_INTERVAL_MAX = 0.5 DEFAULT_CELERY_INTERVAL_STEP = 0.5 -# legacy setting for old sites who still use a local_settings.py file and have -# not updated to presets/ -try: - from local_settings import * # noqa -except ImportError: - pass - # email verification ENABLE_EMAIL_VERIFICATION = False -VERIFIED_KEY_TEXT = 'ALREADY_ACTIVATED' +VERIFIED_KEY_TEXT = "ALREADY_ACTIVATED" -XLS_EXTENSIONS = ['xls', 'xlsx'] +XLS_EXTENSIONS = ["xls", "xlsx"] -CSV_EXTENSION = 'csv' +CSV_EXTENSION = "csv" PROJECT_QUERY_CHUNK_SIZE = 5000 # Prevents "The number of GET/POST parameters exceeded" exception DATA_UPLOAD_MAX_NUMBER_FIELDS = 10000000 -SECRET_KEY = 'mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j' +SECRET_KEY = "mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j" # Time in minutes to lock out user from account LOCKOUT_TIME = 30 * 60 @@ -612,3 +591,19 @@ def configure_logging(logger, **kwargs): XFORM_SUBMISSION_STAT_CACHE_TIME = 600 XFORM_CHARTS_CACHE_TIME = 600 + +SLAVE_DATABASES = [] + +# Google Export settings +GOOGLE_FLOW = { + "web": { + "client_id": "", + "project_id": "", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_secret": "", + "redirect_uris": [], + "javascript_origins": [] + } +} diff --git a/onadata/settings/debug_toolbar_settings.py b/onadata/settings/debug_toolbar_settings.py index 986ee81421..60c71476e3 100644 --- a/onadata/settings/debug_toolbar_settings.py +++ b/onadata/settings/debug_toolbar_settings.py @@ -1,33 +1,37 @@ +""" +Django debug toolbar example settings module. +""" # this system uses structured settings.py as defined in # http://www.slideshare.net/jacobian/the-best-and-worst-of-django # -# this third-level staging file overrides some definitions in staging.py +# this third-level staging file overrides some definitions in staging_example.py # You may wish to alter it to agree with your local environment # # get most settings from staging_example.py (which in turn, imports from # settings.py) # flake8: noqa -from onadata.settings.common import * + +from onadata.settings.common import * # noqa pylint: disable=W0401,W0614 # # # now override the settings which came from staging # # # # DATABASES = { - 'default': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': 'onadata', - 'USER': 'onadata', - 'PASSWORD': '', - 'HOST': '127.0.0.1' + "default": { + "ENGINE": "django.contrib.gis.db.backends.postgis", + "NAME": "onadata", + "USER": "onadata", + "PASSWORD": "", + "HOST": "127.0.0.1", } } DATABASE_ROUTERS = [] # turn off second database # Make a unique unique key just for testing, and don't share it with anybody. -SECRET_KEY = 'mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j' +SECRET_KEY = "mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j" DEBUG = True -INSTALLED_APPS += ('debug_toolbar', ) -REST_FRAMEWORK['DEFAULT_RENDERER_CLASSES'] += ( - 'onadata.libs.renderers.renderers.DebugToolbarRenderer', +INSTALLED_APPS += ("debug_toolbar",) +REST_FRAMEWORK["DEFAULT_RENDERER_CLASSES"] += ( + "onadata.libs.renderers.renderers.DebugToolbarRenderer", ) diff --git a/onadata/settings/default_settings.py b/onadata/settings/default_settings.py index ae50178d52..16669f7741 100644 --- a/onadata/settings/default_settings.py +++ b/onadata/settings/default_settings.py @@ -1,25 +1,29 @@ +# -*- coding: utf-8 -*- +""" +Default settings module. +""" # this system uses structured settings.py as defined in # http://www.slideshare.net/jacobian/the-best-and-worst-of-django # -# this third-level staging file overrides some definitions in staging.py +# this third-level staging file overrides some definitions in staging_example.py # You may wish to alter it to agree with your local environment # # get most settings from staging_example.py (which in turn, imports from # settings.py) -from onadata.settings.staging_example import * # noqa +from onadata.settings.staging_example import * # noqa pylint: disable=W0401,W0614 -ALLOWED_HOSTS = ['*'] +ALLOWED_HOSTS = ["*"] # # # now override the settings which came from staging # # # # DATABASES = { - 'default': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': 'onadata', - 'USER': 'onadata', - 'PASSWORD': '', - 'HOST': '127.0.0.1' + "default": { + "ENGINE": "django.contrib.gis.db.backends.postgis", + "NAME": "onadata", + "USER": "onadata", + "PASSWORD": "", + "HOST": "127.0.0.1", } } @@ -27,4 +31,4 @@ SLAVE_DATABASES = [] # Make a unique unique key just for testing, and don't share it with anybody. -SECRET_KEY = 'mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j' +SECRET_KEY = "mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j" diff --git a/onadata/settings/docker.py b/onadata/settings/docker.py index 3c80d21c48..558b73635a 100644 --- a/onadata/settings/docker.py +++ b/onadata/settings/docker.py @@ -15,17 +15,17 @@ import subprocess import sys -from onadata.settings.common import * # noqa +from onadata.settings.common import * # noqa pylint: disable=W0401,W0614 # # # now override the settings which came from staging # # # # DATABASES = { - 'default': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': 'onadata', - 'USER': 'onadata', - 'PASSWORD': 'onadata', - 'HOST': 'db', - 'PORT': 5432 + "default": { + "ENGINE": "django.contrib.gis.db.backends.postgis", + "NAME": "onadata", + "USER": "onadata", + "PASSWORD": "onadata", + "HOST": "db", + "PORT": 5432, } } @@ -33,11 +33,11 @@ SLAVE_DATABASES = [] # Make a unique unique key just for testing, and don't share it with anybody. -SECRET_KEY = '~&nN9d`bxmJL2[$HhYE9qAk=+4P:cf3b' +SECRET_KEY = "~&nN9d`bxmJL2[$HhYE9qAk=+4P:cf3b" -ALLOWED_HOSTS = ['127.0.0.1', 'localhost'] +ALLOWED_HOSTS = ["127.0.0.1", "localhost"] -INTERNAL_IPS = ['127.0.0.1'] +INTERNAL_IPS = ["127.0.0.1"] DEBUG = True CORS_ORIGIN_ALLOW_ALL = True @@ -50,61 +50,58 @@ else: TESTING_MODE = False -CELERY_BROKER_URL = 'redis://queue:6379' -CELERY_RESULT_BACKEND = 'redis://queue:6379' +CELERY_BROKER_URL = "redis://queue:6379" +CELERY_RESULT_BACKEND = "redis://queue:6379" CELERY_TASK_ALWAYS_EAGER = True -CELERY_ACCEPT_CONTENT = ['json'] -CELERY_TASK_SERIALIZER = 'json' -CELERY_CACHE_BACKEND = 'memory' +CELERY_ACCEPT_CONTENT = ["json"] +CELERY_TASK_SERIALIZER = "json" +CELERY_CACHE_BACKEND = "memory" CELERY_BROKER_CONNECTION_MAX_RETRIES = 2 CACHES = { - 'default': { - 'BACKEND': 'django_redis.cache.RedisCache', - 'LOCATION': 'redis://queue:6379', - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient" - }, + "default": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": "redis://queue:6379", + "OPTIONS": {"CLIENT_CLASS": "django_redis.client.DefaultClient"}, } } NOTIFICATION_BACKENDS = { - 'mqtt': { - 'BACKEND': 'onadata.apps.messaging.backends.mqtt.MQTTBackend', - 'OPTIONS': { - 'HOST': 'notifications', - 'PORT': 1883, - 'QOS': 1, - 'RETAIN': False, - 'SECURE': False, - 'TOPIC_BASE': 'onadata' - } + "mqtt": { + "BACKEND": "onadata.apps.messaging.backends.mqtt.MQTTBackend", + "OPTIONS": { + "HOST": "notifications", + "PORT": 1883, + "QOS": 1, + "RETAIN": False, + "SECURE": False, + "TOPIC_BASE": "onadata", + }, } } FULL_MESSAGE_PAYLOAD = True -EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' +EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" if TESTING_MODE: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'test_media/') # noqa + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "test_media/") # noqa subprocess.call(["rm", "-r", MEDIA_ROOT]) # need to have TASK_ALWAYS_EAGERY True and BROKER_URL as memory # to run tasks immediately while testing CELERY_TASK_ALWAYS_EAGER = True - CELERY_RESULT_BACKEND = 'cache' - CELERY_CACHE_BACKEND = 'memory' - ENKETO_API_TOKEN = 'abc' - ENKETO_PROTOCOL = 'https' - ENKETO_URL = 'https://enketo.ona.io/' - ENKETO_API_ALL_SURVEY_LINKS_PATH = '/api_v2/survey' - ENKETO_API_INSTANCE_PATH = '/api_v1/instance' - ENKETO_SINGLE_SUBMIT_PATH = '/api/v2/survey/single/once' + CELERY_RESULT_BACKEND = "cache" + CELERY_CACHE_BACKEND = "memory" + ENKETO_API_TOKEN = "abc" + ENKETO_PROTOCOL = "https" + ENKETO_URL = "https://enketo.ona.io/" + ENKETO_API_ALL_SURVEY_LINKS_PATH = "/api_v2/survey" + ENKETO_API_INSTANCE_PATH = "/api_v1/instance" + ENKETO_SINGLE_SUBMIT_PATH = "/api/v2/survey/single/once" ENKETO_API_INSTANCE_IFRAME_URL = ENKETO_URL + "api_v1/instance/iframe" NOTIFICATION_BACKENDS = {} else: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'media/') # noqa + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "media/") # noqa -ENKETO_API_ALL_SURVEY_LINKS_PATH = '/api_v2/survey/all' +ENKETO_API_ALL_SURVEY_LINKS_PATH = "/api_v2/survey/all" SUBMISSION_RETRIEVAL_THRESHOLD = 1000 CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD = 100000 - diff --git a/onadata/settings/drone_test.py b/onadata/settings/drone_test.py index bd14196e01..b611bab2cd 100644 --- a/onadata/settings/drone_test.py +++ b/onadata/settings/drone_test.py @@ -1,42 +1,49 @@ +# -*- coding:utf-8 -*- +""" +Example local_settings.py for use with DroneCI. +""" # flake8: noqa # this preset is used for automated testing of formhub # -from onadata.settings.common import * +import subprocess + +from onadata.settings.common import * # noqa pylint: disable=W0401,W0614 DATABASES = { - 'default': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': 'onadata_test', - 'USER': 'postgres', - 'PASSWORD': '', - 'HOST': '127.0.0.1' + "default": { + "ENGINE": "django.contrib.gis.db.backends.postgis", + "NAME": "onadata_test", + "USER": "postgres", + "PASSWORD": "", + "HOST": "127.0.0.1", } } -SECRET_KEY = 'mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j' +SECRET_KEY = "please replace this text" PASSWORD_HASHERS = ( - 'django.contrib.auth.hashers.MD5PasswordHasher', - 'django.contrib.auth.hashers.SHA1PasswordHasher', + "django.contrib.auth.hashers.MD5PasswordHasher", + "django.contrib.auth.hashers.SHA1PasswordHasher", ) +DEBUG = True + if PRINT_EXCEPTION and DEBUG: - MIDDLEWARE_CLASSES += ('utils.middleware.ExceptionLoggingMiddleware',) + MIDDLEWARE += ("utils.middleware.ExceptionLoggingMiddleware",) -if len(sys.argv) >= 2 and (sys.argv[1] == "test" or sys.argv[1] == "test_all"): - # This trick works only when we run tests from the command line. - TESTING_MODE = True -else: - TESTING_MODE = False +# This trick works only when we run tests from the command line. +TESTING_MODE = len(sys.argv) >= 2 and ( + sys.argv[1] == "test" or sys.argv[1] == "test_all" +) if TESTING_MODE: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'test_media/') + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "test_media/") subprocess.call(["rm", "-r", MEDIA_ROOT]) # need to have CELERY_TASK_ALWAYS_EAGER True and BROKER_BACKEND as memory # to run tasks immediately while testing CELERY_TASK_ALWAYS_EAGER = True - CELERY_RESULT_BACKEND = 'cache' - CELERY_CACHE_BACKEND = 'memory' - ENKETO_API_TOKEN = 'abc' + CELERY_RESULT_BACKEND = "cache" + CELERY_CACHE_BACKEND = "memory" + ENKETO_API_TOKEN = "abc" else: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'media/') + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "media/") diff --git a/onadata/settings/github_actions_test.py b/onadata/settings/github_actions_test.py index 7f298106e2..2346b71722 100644 --- a/onadata/settings/github_actions_test.py +++ b/onadata/settings/github_actions_test.py @@ -1,70 +1,70 @@ +# -*- coding: utf-8 -*- +""" +Django settings module or use on GitHub actions. +""" # flake8: noqa # this preset is used for automated testing of onadata from __future__ import absolute_import -from .common import * # nopep8 +from onadata.settings.common import * # noqa pylint: disable=W0401,W0614 # database settings DATABASES = { - 'default': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': 'onadata', - 'USER': 'onadata', - 'PASSWORD': 'onadata', - 'HOST': 'localhost' + "default": { + "ENGINE": "django.contrib.gis.db.backends.postgis", + "NAME": "onadata", + "USER": "onadata", + "PASSWORD": "onadata", + "HOST": "localhost", } } SLAVE_DATABASES = [] -SECRET_KEY = 'mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j' # nosec +SECRET_KEY = "mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j" # nosec -JWT_SECRET_KEY = 'thesecretkey' # nosec -JWT_ALGORITHM = 'HS256' +JWT_SECRET_KEY = "thesecretkey" # nosec +JWT_ALGORITHM = "HS256" -if len(sys.argv) >= 2 and (sys.argv[1] == "test" or sys.argv[1] == "test_all"): - # This trick works only when we run tests from the command line. - TESTING_MODE = True -else: - TESTING_MODE = False +# This trick works only when we run tests from the command line. +TESTING_MODE = len(sys.argv) >= 2 and ( + sys.argv[1] == "test" or sys.argv[1] == "test_all" +) if TESTING_MODE: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'test_media/') + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "test_media/") # subprocess.call(["rm", "-r", MEDIA_ROOT]) # nosec # need to have CELERY_TASK_ALWAYS_EAGER True and BROKER_BACKEND as memory # to run tasks immediately while testing CELERY_BROKER_URL = "memory://" CELERY_TASK_ALWAYS_EAGER = True - CELERY_RESULT_BACKEND = 'cache' - CELERY_CACHE_BACKEND = 'memory' - ENKETO_API_TOKEN = 'abc' # nosec - ENKETO_PROTOCOL = 'https' - ENKETO_URL = 'https://enketo.ona.io/' - ENKETO_API_ALL_SURVEY_LINKS_PATH = '/api_v2/survey/all' - ENKETO_API_INSTANCE_PATH = '/api_v1/instance' - ENKETO_SINGLE_SUBMIT_PATH = '/api/v2/survey/single/once' + CELERY_RESULT_BACKEND = "cache" + CELERY_CACHE_BACKEND = "memory" + ENKETO_API_TOKEN = "abc" # nosec + ENKETO_PROTOCOL = "https" + ENKETO_URL = "https://enketo.ona.io/" + ENKETO_API_ALL_SURVEY_LINKS_PATH = "/api_v2/survey/all" + ENKETO_API_INSTANCE_PATH = "/api_v1/instance" + ENKETO_SINGLE_SUBMIT_PATH = "/api/v2/survey/single/once" ENKETO_API_INSTANCE_IFRAME_URL = ENKETO_URL + "api_v1/instance/iframe" else: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'media/') + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "media/") -PASSWORD_HASHERS = ( - 'django.contrib.auth.hashers.MD5PasswordHasher', -) +PASSWORD_HASHERS = ("django.contrib.auth.hashers.MD5PasswordHasher",) DEBUG = False -TEMPLATES[0]['OPTIONS']['debug'] = DEBUG +TEMPLATES[0]["OPTIONS"]["debug"] = DEBUG MIDDLEWARE_CLASSES = ( - 'django.middleware.common.CommonMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', + "django.middleware.common.CommonMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", # 'django.middleware.locale.LocaleMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'onadata.libs.utils.middleware.HTTPResponseNotAllowedMiddleware', + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "onadata.libs.utils.middleware.HTTPResponseNotAllowedMiddleware", ) -VERIFIED_KEY_TEXT = 'ALREADY_ACTIVATED' +VERIFIED_KEY_TEXT = "ALREADY_ACTIVATED" -ODK_TOKEN_FERNET_KEY = 'ROsB4T8s1rCJskAdgpTQEKfH2x2K_EX_YBi3UFyoYng=' # nosec +ODK_TOKEN_FERNET_KEY = "ROsB4T8s1rCJskAdgpTQEKfH2x2K_EX_YBi3UFyoYng=" # nosec OPENID_CONNECT_PROVIDERS = {} - diff --git a/onadata/settings/production_example.py b/onadata/settings/production_example.py index 0aa7eb1bd4..ea57dcf820 100644 --- a/onadata/settings/production_example.py +++ b/onadata/settings/production_example.py @@ -1,5 +1,10 @@ +# -*- coding=utf-8 -*- +""" +Example local_settings.py used by the Dockerfile. +""" # flake8: noqa -from onadata.settings.common import * + +from onadata.settings.common import * # noqa pylint: disable=W0401,W0614 # this setting file will not work on "runserver" -- it needs a server for # static files @@ -7,33 +12,31 @@ # override to set the actual location for the production static and media # directories -MEDIA_ROOT = '/var/formhub-media' +MEDIA_ROOT = "/var/formhub-media" STATIC_ROOT = "/srv/formhub-static" -STATICFILES_DIRS = ( - os.path.join(PROJECT_ROOT, "static"), -) +STATICFILES_DIRS = (os.path.join(PROJECT_ROOT, "static"),) ADMINS = ( # ('Your Name', 'your_email@example.com'), ) # your actual production settings go here...,. DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'formhub', - 'USER': 'formhub_prod', + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": "formhub", + "USER": "formhub_prod", # the password must be stored in an environment variable - 'PASSWORD': os.environ['FORMHUB_PROD_PW'], + "PASSWORD": os.environ["FORMHUB_PROD_PW"], # the server name may be in env - 'HOST': os.environ.get("FORMHUB_DB_SERVER", 'dbserver.yourdomain.org') + "HOST": os.environ.get("FORMHUB_DB_SERVER", "dbserver.yourdomain.org"), }, - 'gis': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': 'phis', - 'USER': 'staff', + "gis": { + "ENGINE": "django.contrib.gis.db.backends.postgis", + "NAME": "phis", + "USER": "staff", # the password must be stored in an environment variable - 'PASSWORD': os.environ['PHIS_PW'], - 'HOST': 'gisserver.yourdomain.org' - } + "PASSWORD": os.environ["PHIS_PW"], + "HOST": "gisserver.yourdomain.org", + }, } # Local time zone for this installation. Choices can be found here: @@ -43,36 +46,36 @@ # timezone as the operating system. # If running in a Windows environment this must be set to the same as your # system time zone. -TIME_ZONE = 'Africa/Lagos' +TIME_ZONE = "Africa/Lagos" -TOUCHFORMS_URL = 'http://localhost:9000/' +TOUCHFORMS_URL = "http://localhost:9000/" # Make this unique, and don't share it with anybody. -SECRET_KEY = 'mlfs33^s1l4xf6a36$0#j%dd*sisfo6HOktYXB9y' +SECRET_KEY = "mlfs33^s1l4xf6a36$0#j%dd*sisfo6HOktYXB9y" # Caching CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', - 'LOCATION': '127.0.0.1:11211', + "default": { + "BACKEND": "django.core.cache.backends.memcached.PyLibMCCache", + "LOCATION": "127.0.0.1:11211", } } -MIDDLEWARE_CLASSES += ('django.middleware.cache.UpdateCacheMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.cache.FetchFromCacheMiddleware',) +MIDDLEWARE += ( + "django.middleware.cache.UpdateCacheMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.cache.FetchFromCacheMiddleware", +) CACHE_MIDDLEWARE_SECONDS = 3600 # 1 hour -CACHE_MIDDLEWARE_KEY_PREFIX = '' +CACHE_MIDDLEWARE_KEY_PREFIX = "" REST_SERVICES_TO_MODULES = { - 'google_sheets': 'google_export.services', + "google_sheets": "google_export.services", } REST_SERVICES_TO_SERIALIZERS = { - 'google_sheets': 'google_export.serializers.GoogleSheetsSerializer' + "google_sheets": "google_export.serializers.GoogleSheetsSerializer" } -CUSTOM_MAIN_URLS = { - 'google_export.urls' -} +CUSTOM_MAIN_URLS = {"google_export.urls"} diff --git a/onadata/settings/staging_example.py b/onadata/settings/staging_example.py index fe7b2f1f27..9436355644 100644 --- a/onadata/settings/staging_example.py +++ b/onadata/settings/staging_example.py @@ -1,45 +1,50 @@ -# flake8: noqa -from onadata.settings.common import * +# -*- coding: utf-8 -*- +""" +Example staging module. +""" +import os +import subprocess +import sys + +from onadata.settings.common import * # noqa pylint: disable=W0401,W0614 DEBUG = True -TEMPLATES[0]['OPTIONS']['debug'] = DEBUG -TEMPLATE_STRING_IF_INVALID = '' +TEMPLATES[0]["OPTIONS"]["debug"] = DEBUG # noqa +TEMPLATE_STRING_IF_INVALID = "" # see: http://docs.djangoproject.com/en/dev/ref/settings/#databases # postgres DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'formhub_dev', - 'USER': 'formhub_dev', - 'PASSWORD': '12345678', - 'HOST': 'localhost' + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": "formhub_dev", + "USER": "formhub_dev", + "PASSWORD": "12345678", + "HOST": "localhost", }, } # TIME_ZONE = 'UTC' -SECRET_KEY = 'mlfs33^s1l4xf6a36$0#srgcpj%dd*sisfo6HOktYXB9y' +SECRET_KEY = "please replace this text" -TESTING_MODE = False -if len(sys.argv) >= 2 and (sys.argv[1] == "test" or sys.argv[1] == "test_all"): - # This trick works only when we run tests from the command line. - TESTING_MODE = True -else: - TESTING_MODE = False +# This trick works only when we run tests from the command line. +TESTING_MODE = len(sys.argv) >= 2 and ( + sys.argv[1] == "test" or sys.argv[1] == "test_all" +) if TESTING_MODE: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'test_media/') - subprocess.call(["rm", "-r", MEDIA_ROOT]) + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "test_media/") # noqa + subprocess.call(["rm", "-r", MEDIA_ROOT]) # noqa # need to have CELERY_TASK_ALWAYS_EAGER True and BROKER_BACKEND as memory # to run tasks immediately while testing CELERY_TASK_ALWAYS_EAGER = True - CELERY_RESULT_BACKEND = 'cache' - CELERY_CACHE_BACKEND = 'memory' - ENKETO_API_TOKEN = 'abc' + CELERY_RESULT_BACKEND = "cache" + CELERY_CACHE_BACKEND = "memory" + ENKETO_API_TOKEN = "abc" else: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'media/') + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "media/") # noqa -if PRINT_EXCEPTION and DEBUG: - MIDDLEWARE_CLASSES += ('utils.middleware.ExceptionLoggingMiddleware',) +if PRINT_EXCEPTION and DEBUG: # noqa + MIDDLEWARE += ("utils.middleware.ExceptionLoggingMiddleware",) # noqa diff --git a/onadata/settings/travis_test.py b/onadata/settings/travis_test.py index 4bda303111..98f75cb8e3 100644 --- a/onadata/settings/travis_test.py +++ b/onadata/settings/travis_test.py @@ -1,69 +1,73 @@ +# -*- coding: utf-8 -*- +""" +Django settings module or use on GitHub actions. +""" # flake8: noqa -# this preset is used for automated testing of formhub +# this preset is used for automated testing of onadata from __future__ import absolute_import -from .common import * # nopep8 +import subprocess + +from onadata.settings.common import * # noqa pylint: disable=W0401,W0614 + # database settings DATABASES = { - 'default': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': 'onadata_test', - 'USER': 'postgres', - 'PASSWORD': '', - 'HOST': '127.0.0.1' + "default": { + "ENGINE": "django.contrib.gis.db.backends.postgis", + "NAME": "onadata_test", + "USER": "postgres", + "PASSWORD": "", + "HOST": "127.0.0.1", } } SLAVE_DATABASES = [] -SECRET_KEY = 'mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j' +SECRET_KEY = "mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j" -JWT_SECRET_KEY = 'thesecretkey' -JWT_ALGORITHM = 'HS256' +JWT_SECRET_KEY = "thesecretkey" +JWT_ALGORITHM = "HS256" -if len(sys.argv) >= 2 and (sys.argv[1] == "test" or sys.argv[1] == "test_all"): - # This trick works only when we run tests from the command line. - TESTING_MODE = True -else: - TESTING_MODE = False +# This trick works only when we run tests from the command line. +TESTING_MODE = len(sys.argv) >= 2 and ( + sys.argv[1] == "test" or sys.argv[1] == "test_all" +) if TESTING_MODE: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'test_media/') + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "test_media/") subprocess.call(["rm", "-r", MEDIA_ROOT]) # need to have CELERY_TASK_ALWAYS_EAGER True and BROKER_BACKEND as memory # to run tasks immediately while testing CELERY_BROKER_URL = "memory://" CELERY_TASK_ALWAYS_EAGER = True - CELERY_RESULT_BACKEND = 'cache' - CELERY_CACHE_BACKEND = 'memory' - ENKETO_API_TOKEN = 'abc' - ENKETO_PROTOCOL = 'https' - ENKETO_URL = 'https://enketo.ona.io/' - ENKETO_API_ALL_SURVEY_LINKS_PATH = '/api_v2/survey/all' - ENKETO_API_INSTANCE_PATH = '/api_v1/instance' - ENKETO_SINGLE_SUBMIT_PATH = '/api/v2/survey/single/once' + CELERY_RESULT_BACKEND = "cache" + CELERY_CACHE_BACKEND = "memory" + ENKETO_API_TOKEN = "abc" + ENKETO_PROTOCOL = "https" + ENKETO_URL = "https://enketo.ona.io/" + ENKETO_API_ALL_SURVEY_LINKS_PATH = "/api_v2/survey/all" + ENKETO_API_INSTANCE_PATH = "/api_v1/instance" + ENKETO_SINGLE_SUBMIT_PATH = "/api/v2/survey/single/once" ENKETO_API_INSTANCE_IFRAME_URL = ENKETO_URL + "api_v1/instance/iframe" else: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'media/') + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "media/") -PASSWORD_HASHERS = ( - 'django.contrib.auth.hashers.MD5PasswordHasher', -) +PASSWORD_HASHERS = ("django.contrib.auth.hashers.MD5PasswordHasher",) DEBUG = False -TEMPLATES[0]['OPTIONS']['debug'] = DEBUG -MIDDLEWARE_CLASSES = ( - 'django.middleware.common.CommonMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', +TEMPLATES[0]["OPTIONS"]["debug"] = DEBUG +MIDDLEWARE = ( + "django.middleware.common.CommonMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", # 'django.middleware.locale.LocaleMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'onadata.libs.utils.middleware.HTTPResponseNotAllowedMiddleware', + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "onadata.libs.utils.middleware.HTTPResponseNotAllowedMiddleware", ) -VERIFIED_KEY_TEXT = 'ALREADY_ACTIVATED' +VERIFIED_KEY_TEXT = "ALREADY_ACTIVATED" -ODK_TOKEN_FERNET_KEY = 'ROsB4T8s1rCJskAdgpTQEKfH2x2K_EX_YBi3UFyoYng=' +ODK_TOKEN_FERNET_KEY = "ROsB4T8s1rCJskAdgpTQEKfH2x2K_EX_YBi3UFyoYng=" OPENID_CONNECT_PROVIDERS = {} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..fdad2c0aba --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,8 @@ +[build-system] +requires = [ + "setuptools >= 48", + "setuptools_scm[toml] >= 4, <6", + "setuptools_scm_git_archive", + "wheel >= 0.29.0", +] +build-backend = 'setuptools.build_meta' diff --git a/requirements/base.in b/requirements/base.in index 5b015c0d3c..c2732842d0 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -2,10 +2,9 @@ -e . # installed from Git --e git+https://github.com/onaio/python-digest.git@3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest --e git+https://github.com/onaio/django-digest.git@eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest +-e git+https://github.com/onaio/python-digest.git@08267ca8afc1a52f91352ebb5385e8e6d074fc36#egg=python-digest +-e git+https://github.com/onaio/django-digest.git@6bf61ec08502fd3545d4f2c0838b6cb15e7ffa92#egg=django-digest -e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router -e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip -e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient --e git+https://github.com/onaio/oauth2client.git@75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client -e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc diff --git a/requirements/base.pip b/requirements/base.pip index af8f32d28a..04a709a010 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -1,59 +1,66 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile --output-file=requirements/base.pip requirements/base.in # --e git+https://github.com/onaio/django-digest.git@eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest +-e git+https://github.com/onaio/django-digest.git@6bf61ec08502fd3545d4f2c0838b6cb15e7ffa92#egg=django-digest # via -r requirements/base.in -e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router # via -r requirements/base.in --e git+https://github.com/onaio/oauth2client.git@75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client - # via -r requirements/base.in -e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc # via -r requirements/base.in -e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip # via -r requirements/base.in --e git+https://github.com/onaio/python-digest.git@3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest +-e git+https://github.com/onaio/python-digest.git@08267ca8afc1a52f91352ebb5385e8e6d074fc36#egg=python-digest # via -r requirements/base.in -e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient # via -r requirements/base.in alabaster==0.7.12 # via sphinx -amqp==5.0.6 +amqp==5.1.1 # via kombu -analytics-python==1.3.1 +analytics-python==1.4.0 # via onadata appoptics-metrics==5.1.0 # via onadata -attrs==21.2.0 - # via jsonschema -babel==2.9.1 +asgiref==3.5.1 + # via django +async-timeout==4.0.2 + # via redis +attrs==21.4.0 + # via + # jsonlines + # jsonschema +babel==2.10.1 # via sphinx backoff==1.10.0 # via analytics-python billiard==3.6.4.0 # via celery -boto3==1.17.74 +boto3==1.22.7 # via tabulator -botocore==1.20.74 +botocore==1.25.7 # via # boto3 # s3transfer cached-property==1.5.2 # via tableschema -celery==5.0.5 +cachetools==5.0.0 + # via google-auth +celery==5.2.6 # via onadata -certifi==2020.12.5 +certifi==2021.10.8 # via requests -cffi==1.14.5 +cffi==1.15.0 # via cryptography chardet==4.0.0 # via # datapackage - # requests # tabulator -click==7.1.2 +charset-normalizer==2.0.12 + # via requests +click==8.1.3 # via # celery # click-didyoumean @@ -62,13 +69,13 @@ click==7.1.2 # datapackage # tableschema # tabulator -click-didyoumean==0.0.3 +click-didyoumean==0.3.0 # via celery click-plugins==1.1.1 # via celery -click-repl==0.1.6 +click-repl==0.2.0 # via celery -cryptography==3.4.7 +cryptography==37.0.2 # via # jwcrypto # onadata @@ -78,12 +85,16 @@ datapackage==1.15.2 defusedxml==0.7.1 # via # djangorestframework-xml + # onadata # pyxform -deprecated==1.2.12 - # via onadata -dict2xml==1.7.0 +deprecated==1.2.13 + # via + # jwcrypto + # onadata + # redis +dict2xml==1.7.1 # via onadata -django==2.2.23 +django==3.2.13 # via # django-cors-headers # django-debug-toolbar @@ -98,42 +109,41 @@ django==2.2.23 # djangorestframework # djangorestframework-guardian # djangorestframework-jsonapi - # jsonfield # ona-oidc # onadata -django-activity-stream==0.10.0 +django-activity-stream==1.4.0 # via onadata -django-cors-headers==3.7.0 +django-cors-headers==3.11.0 # via onadata -django-debug-toolbar==3.2.1 +django-debug-toolbar==3.4.0 # via onadata -django-filter==2.4.0 +django-filter==21.1 # via onadata -django-guardian==2.3.0 +django-guardian==2.4.0 # via # djangorestframework-guardian # onadata django-nose==1.4.7 # via onadata -django-oauth-toolkit==1.5.0 +django-oauth-toolkit==2.0.0 # via onadata -django-ordered-model==3.4.3 +django-ordered-model==3.5 # via onadata django-query-builder==2.0.1 # via onadata -django-redis==5.0.0 +django-redis==5.2.0 # via onadata -django-registration-redux==2.9 +django-registration-redux==2.10 # via onadata -django-render-block==0.8.1 +django-render-block==0.9.1 # via django-templated-email -django-reversion==3.0.9 +django-reversion==5.0.0 # via onadata -django-taggit==1.4.0 +django-taggit==3.0.0 # via onadata -django-templated-email==2.3.0 +django-templated-email==3.0.0 # via onadata -djangorestframework==3.12.4 +djangorestframework==3.13.1 # via # djangorestframework-csv # djangorestframework-gis @@ -143,11 +153,11 @@ djangorestframework==3.12.4 # onadata djangorestframework-csv==2.1.1 # via onadata -djangorestframework-gis==0.17 +djangorestframework-gis==0.18 # via onadata djangorestframework-guardian==0.3.0 # via onadata -djangorestframework-jsonapi==4.2.0 +djangorestframework-jsonapi==5.0.0 # via onadata djangorestframework-jsonp==1.0.2 # via onadata @@ -155,70 +165,74 @@ djangorestframework-xml==2.0.0 # via onadata docutils==0.17.1 # via sphinx -dpath==2.0.1 +dpath==2.0.6 # via onadata elaphe3==0.2.0 # via onadata et-xmlfile==1.1.0 # via openpyxl -flake8==3.9.2 +flake8==4.0.1 # via onadata -fleming==0.6.0 +fleming==0.7.0 # via django-query-builder future==0.18.2 # via python-json2xlsclient geojson==2.5.0 # via onadata -greenlet==1.1.0 +google-auth==2.6.6 + # via + # google-auth-oauthlib + # onadata +google-auth-oauthlib==0.5.1 + # via onadata +greenlet==1.1.2 # via sqlalchemy httmock==1.4.0 # via onadata -httplib2==0.19.1 - # via - # oauth2client - # onadata -idna==2.10 +httplib2==0.20.4 + # via onadata +idna==3.3 # via requests ijson==3.1.4 # via tabulator -imagesize==1.2.0 +imagesize==1.3.0 # via sphinx +importlib-metadata==4.11.3 + # via + # markdown + # sphinx inflection==0.5.1 # via djangorestframework-jsonapi -isodate==0.6.0 +isodate==0.6.1 # via tableschema -jinja2==2.11.3 +jinja2==3.1.2 # via sphinx -jmespath==0.10.0 +jmespath==1.0.0 # via # boto3 # botocore -jsonfield==0.9.23 - # via onadata -jsonlines==2.0.0 +jsonlines==3.0.0 # via tabulator -jsonpickle==2.0.0 +jsonpickle==2.1.0 # via onadata -jsonpointer==2.1 +jsonpointer==2.3 # via datapackage -jsonschema==3.2.0 +jsonschema==4.4.0 # via # datapackage # tableschema -jwcrypto==0.8 +jwcrypto==1.2 # via django-oauth-toolkit -kombu==5.0.2 +kombu==5.2.4 # via celery linear-tsv==1.1.0 # via tabulator -lxml==4.6.3 +lxml==4.8.0 # via onadata -markdown==3.3.4 +markdown==3.3.6 # via onadata -markupsafe==1.1.1 - # via - # jinja2 - # sphinx +markupsafe==2.1.1 + # via jinja2 mccabe==0.6.1 # via flake8 mock==4.0.3 @@ -229,57 +243,60 @@ monotonic==1.6 # via analytics-python nose==1.3.7 # via django-nose -numpy==1.19.5 +numpy==1.22.3 # via onadata -oauthlib==3.1.0 - # via django-oauth-toolkit +oauthlib==3.2.0 + # via + # django-oauth-toolkit + # requests-oauthlib openpyxl==3.0.9 # via # onadata # pyxform # tabulator -packaging==20.9 - # via sphinx -paho-mqtt==1.5.1 +packaging==21.3 + # via + # redis + # sphinx +paho-mqtt==1.6.1 # via onadata -pillow==8.2.0 +pillow==9.1.0 # via # elaphe3 # onadata -prompt-toolkit==3.0.18 +prompt-toolkit==3.0.29 # via click-repl -psycopg2==2.8.6 +psycopg2-binary==2.9.3 # via onadata pyasn1==0.4.8 # via - # oauth2client # pyasn1-modules # rsa pyasn1-modules==0.2.8 - # via oauth2client -pycodestyle==2.7.0 + # via google-auth +pycodestyle==2.8.0 # via flake8 -pycparser==2.20 +pycparser==2.21 # via cffi -pyflakes==2.3.1 +pyflakes==2.4.0 # via flake8 -pygments==2.9.0 +pygments==2.12.0 # via sphinx -pyjwt[crypto]==2.1.0 +pyjwt[crypto]==2.3.0 # via # ona-oidc # onadata pylibmc==1.6.1 # via onadata -pymongo==3.11.4 +pymongo==4.1.1 # via onadata -pyparsing==2.4.7 +pyparsing==3.0.8 # via # httplib2 # packaging -pyrsistent==0.17.3 +pyrsistent==0.18.1 # via jsonschema -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # analytics-python # botocore @@ -288,12 +305,13 @@ python-dateutil==2.8.1 # tableschema python-memcached==1.59 # via onadata -pytz==2021.1 +pytz==2022.1 # via # babel # celery # django # django-query-builder + # djangorestframework # fleming # onadata pyxform==1.10.0 @@ -304,9 +322,9 @@ raven==6.10.0 # via onadata recaptcha-client==1.0.6 # via onadata -redis==3.5.3 +redis==4.2.2 # via django-redis -requests==2.25.1 +requests==2.27.1 # via # analytics-python # datapackage @@ -316,20 +334,23 @@ requests==2.25.1 # onadata # python-json2xlsclient # requests-mock + # requests-oauthlib # sphinx # tableschema # tabulator -requests-mock==1.9.2 +requests-mock==1.9.3 # via onadata -rfc3986==1.5.0 +requests-oauthlib==1.3.1 + # via google-auth-oauthlib +rfc3986==2.0.0 # via tableschema -rsa==4.7.2 - # via oauth2client -s3transfer==0.4.2 +rsa==4.8 + # via google-auth +s3transfer==0.5.2 # via boto3 savreaderwriter==3.4.2 # via onadata -simplejson==3.17.2 +simplejson==3.17.6 # via onadata six==1.16.0 # via @@ -337,38 +358,35 @@ six==1.16.0 # appoptics-metrics # click-repl # datapackage - # django-oauth-toolkit # django-query-builder - # django-templated-email # djangorestframework-csv + # google-auth # isodate - # jsonschema # linear-tsv - # oauth2client # python-dateutil # python-memcached # requests-mock # tableschema # tabulator -snowballstemmer==2.1.0 +snowballstemmer==2.2.0 # via sphinx -sphinx==4.0.1 +sphinx==4.5.0 # via onadata sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx -sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-htmlhelp==2.0.0 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.3 # via sphinx -sphinxcontrib-serializinghtml==1.1.4 +sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy==1.4.15 +sqlalchemy==1.4.36 # via tabulator -sqlparse==0.4.1 +sqlparse==0.4.2 # via # django # django-debug-toolbar @@ -385,29 +403,28 @@ unicodecsv==0.14.1 # onadata # tableschema # tabulator -urllib3==1.26.4 +urllib3==1.26.9 # via # botocore # requests -uwsgi==2.0.19.1 +uwsgi==2.0.20 # via onadata vine==5.0.0 # via # amqp # celery + # kombu wcwidth==0.2.5 # via prompt-toolkit -wrapt==1.12.1 +wrapt==1.14.1 # via deprecated xlrd==2.0.1 # via - # onadata # pyxform # tabulator xlwt==1.3.0 # via onadata xmltodict==0.12.0 # via onadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools +zipp==3.8.0 + # via importlib-metadata diff --git a/requirements/dev.pip b/requirements/dev.pip index 4c0ae4b8b9..ce5fbd0853 100644 --- a/requirements/dev.pip +++ b/requirements/dev.pip @@ -91,7 +91,7 @@ deprecated==1.2.12 # via onadata dict2xml==1.7.0 # via onadata -django==2.2.23 +django==3.2.13 # via # django-cors-headers # django-debug-toolbar diff --git a/requirements/s3.in b/requirements/s3.in index 124bacc622..210fbc7957 100644 --- a/requirements/s3.in +++ b/requirements/s3.in @@ -1,3 +1,3 @@ django-storages -django >=2.2.20,<3 +django >=3.2.13,<4 boto3 diff --git a/requirements/s3.pip b/requirements/s3.pip index a5b4dd4de7..9f166104d5 100644 --- a/requirements/s3.pip +++ b/requirements/s3.pip @@ -1,21 +1,23 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile --output-file=requirements/s3.pip requirements/s3.in # +asgiref==3.5.1 + # via django boto3==1.17.74 # via -r requirements/s3.in botocore==1.20.74 # via # boto3 # s3transfer -django-storages==1.11.1 - # via -r requirements/s3.in -django==2.2.23 +django==3.2.13 # via # -r requirements/s3.in # django-storages +django-storages==1.11.1 + # via -r requirements/s3.in jmespath==0.10.0 # via # boto3 diff --git a/requirements/ses.in b/requirements/ses.in index 3d549791cd..e23a56f077 100644 --- a/requirements/ses.in +++ b/requirements/ses.in @@ -1,3 +1,3 @@ boto -django >=2.2.20,<3 +django >=3.2.13,<4 django-ses diff --git a/requirements/ses.pip b/requirements/ses.pip index ea7f904311..274e3c3653 100644 --- a/requirements/ses.pip +++ b/requirements/ses.pip @@ -1,23 +1,25 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile --output-file=requirements/ses.pip requirements/ses.in # -boto3==1.17.74 - # via django-ses +asgiref==3.5.1 + # via django boto==2.49.0 # via -r requirements/ses.in +boto3==1.17.74 + # via django-ses botocore==1.20.74 # via # boto3 # s3transfer -django-ses==2.0.0 - # via -r requirements/ses.in -django==2.2.23 +django==3.2.13 # via # -r requirements/ses.in # django-ses +django-ses==2.0.0 + # via -r requirements/ses.in future==0.18.2 # via django-ses jmespath==0.10.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000..9b9a5265e6 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,117 @@ +[metadata] +name = onadata +version = 3.0.0 +description = Collect Analyze and Share Data +long_description = file: README.rst +long_description_content_type = text/x-rst +url = https://github.com/onaio/onadata +author = Ona Systems Inc +author_email = support@ona.io +license = Copyright (c) 2022 Ona Systems Inc All rights reserved +license_file = LICENSE +classifiers = + Development Status :: 5 - Production/Stable + Programming Language :: Python :: 3.9 +project_urls = + Documentation = https://api.ona.io/api + Source = https://github.com/onaio/onadata + Tracker = https://github.com/onaio/onadata/issues + +[options] +packages = find: +platforms = any +tests_require = + flake8 + mock + httmock + requests-mock +install_requires = + Django>=3.2.13,<4 + django-guardian + django-registration-redux + django-templated-email + django-reversion + django-filter + django-nose + django-ordered-model + #generic relation + django-query-builder + celery + #cors + django-cors-headers + django-debug-toolbar + #oauth2 support + django-oauth-toolkit + #oauth2client + jsonpickle + #jwt + PyJWT + #captcha + recaptcha-client + #API support + djangorestframework + djangorestframework-csv + djangorestframework-gis + djangorestframework-guardian + djangorestframework-jsonapi + djangorestframework-jsonp + djangorestframework-xml + #geojson + geojson + #tagging + django-taggit + #database + psycopg2-binary>2.7.1 + pymongo + #sms support + dict2xml + lxml + #pyxform + pyxform + #spss + savreaderwriter + #memcached support + pylibmc + python-memcached + #XML Instance API utility + xmltodict + #docs + sphinx + Markdown + #others + unicodecsv + xlwt + openpyxl + dpath + elaphe3 + httplib2 + modilabs-python-utils + numpy + Pillow + python-dateutil + pytz + requests + simplejson + uwsgi + raven + django-activity-stream + paho-mqtt + cryptography + #Monitoring + analytics-python + appoptics-metrics + # Deprecation tagging + deprecated + # Redis cache + django-redis + # osm + defusedxml + # Google exports + google-auth-oauthlib + google-auth +python_requires = >= 3.9 +setup_requires = + setuptools_scm + +[bdist_wheel] +universal = 1 diff --git a/setup.py b/setup.py index cd762c2947..0c93a07ecf 100644 --- a/setup.py +++ b/setup.py @@ -14,121 +14,7 @@ https://ona.io https://opendatakit.org """ +import setuptools -from setuptools import setup, find_packages - -import onadata - -setup( - name="onadata", - version=onadata.__version__, - description="Collect Analyze and Share Data!", - author="Ona Systems Inc", - author_email="support@ona.io", - license="Copyright (c) 2014 Ona Systems Inc All rights reserved.", - project_urls={ - 'Source': 'https://github.com/onaio/onadata', - }, - packages=find_packages(exclude=['docs', 'tests']), - install_requires=[ - "Django>=2.2.20,<3", - "django-guardian", - "django-registration-redux", - "django-templated-email", - "django-reversion", - "django-filter", - "django-nose", - "django-ordered-model", - # generic relation - "django-query-builder", - "celery", - # cors - "django-cors-headers", - "django-debug-toolbar", - # oauth2 support - "django-oauth-toolkit", - # "oauth2client", - "jsonpickle", - # jwt - "PyJWT", - # captcha - "recaptcha-client", - # API support - "djangorestframework", - "djangorestframework-csv", - "djangorestframework-gis", - "djangorestframework-guardian", - "djangorestframework-jsonapi", - "djangorestframework-jsonp", - "djangorestframework-xml", - # geojson - "geojson", - # tagging - "django-taggit", - # database - "psycopg2>2.7.1", - "pymongo", - # sms support - "dict2xml", - "lxml", - # pyxform - "pyxform", - # spss - "savreaderwriter", - # tests - "mock", - "httmock", - # JSON data type support, keeping it around for previous migration - "jsonfield<1.0", - # memcached support - "pylibmc", - "python-memcached", - # XML Instance API utility - "xmltodict", - # docs - "sphinx", - "Markdown", - # others - "unicodecsv", - "xlrd", - "xlwt", - "openpyxl", - "dpath", - "elaphe3", - "httplib2", - "modilabs-python-utils", - "numpy", - "Pillow", - "python-dateutil", - "pytz", - "requests", - "requests-mock", - "simplejson", - "uwsgi", - "flake8", - "raven", - "django-activity-stream", - "paho-mqtt", - "cryptography", - # Monitoring - "analytics-python", - "appoptics-metrics", - # Deprecation tagging - "deprecated", - # Redis cache - "django-redis", - ], - dependency_links=[ - 'https://github.com/onaio/python-digest/tarball/3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest', # noqa pylint: disable=line-too-long - 'https://github.com/onaio/django-digest/tarball/eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest', # noqa pylint: disable=line-too-long - 'https://github.com/onaio/django-multidb-router/tarball/9cf0a0c6c9f796e5bd14637fafeb5a1a5507ed37#egg=django-multidb-router', # noqa pylint: disable=line-too-long - 'https://github.com/onaio/floip-py/tarball/3bbf5c76b34ec49c438a3099ab848870514d1e50#egg=floip', # noqa pylint: disable=line-too-long - 'https://github.com/onaio/python-json2xlsclient/tarball/62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient', # noqa pylint: disable=line-too-long - 'https://github.com/onaio/oauth2client/tarball/75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client', # noqa pylint: disable=line-too-long - ], - extras_require={ - ':python_version=="2.7"': [ - 'functools32>=3.2.3-2' - ] - } -) +if __name__ == "__main__": + setuptools.setup()